2024-11-15 23:14:57,775 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 23:14:57,792 main DEBUG Took 0.013590 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 23:14:57,792 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 23:14:57,793 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 23:14:57,794 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 23:14:57,796 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:14:57,806 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 23:14:57,820 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:14:57,822 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:14:57,822 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:14:57,823 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:14:57,823 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:14:57,823 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:14:57,824 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:14:57,825 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:14:57,825 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:14:57,825 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:14:57,826 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:14:57,826 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:14:57,827 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:14:57,827 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:14:57,828 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:14:57,828 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:14:57,829 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:14:57,829 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:14:57,829 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:14:57,830 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:14:57,830 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:14:57,830 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:14:57,831 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:14:57,831 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:14:57,832 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:14:57,832 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 23:14:57,833 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:14:57,834 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 23:14:57,836 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 23:14:57,837 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 23:14:57,838 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 23:14:57,838 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 23:14:57,845 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 23:14:57,847 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 23:14:57,849 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 23:14:57,849 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 23:14:57,849 main DEBUG createAppenders(={Console}) 2024-11-15 23:14:57,850 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-15 23:14:57,850 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 23:14:57,851 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-15 23:14:57,851 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 23:14:57,851 main DEBUG OutputStream closed 2024-11-15 23:14:57,852 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 23:14:57,852 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 23:14:57,852 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-15 23:14:57,936 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 23:14:57,939 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 23:14:57,941 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 23:14:57,942 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 23:14:57,943 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 23:14:57,944 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 23:14:57,944 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 23:14:57,944 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 23:14:57,945 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 23:14:57,945 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 23:14:57,946 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 23:14:57,946 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 23:14:57,947 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 23:14:57,947 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 23:14:57,948 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 23:14:57,948 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 23:14:57,949 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 23:14:57,950 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 23:14:57,954 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 23:14:57,954 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-15 23:14:57,955 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 23:14:57,956 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-15T23:14:58,236 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f 2024-11-15 23:14:58,240 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 23:14:58,241 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T23:14:58,252 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-15T23:14:58,274 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-15T23:14:58,293 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/cluster_535d8ff1-f3e7-a1c2-84cc-d10f4f48cd95, deleteOnExit=true 2024-11-15T23:14:58,293 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-15T23:14:58,294 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/test.cache.data in system properties and HBase conf 2024-11-15T23:14:58,295 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/hadoop.tmp.dir in system properties and HBase conf 2024-11-15T23:14:58,296 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/hadoop.log.dir in system properties and HBase conf 2024-11-15T23:14:58,297 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-15T23:14:58,297 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-15T23:14:58,298 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-15T23:14:58,389 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T23:14:58,512 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T23:14:58,518 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-15T23:14:58,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-15T23:14:58,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-15T23:14:58,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T23:14:58,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-15T23:14:58,522 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-15T23:14:58,522 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T23:14:58,523 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T23:14:58,524 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-15T23:14:58,524 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/nfs.dump.dir in system properties and HBase conf 2024-11-15T23:14:58,525 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/java.io.tmpdir in system properties and HBase conf 2024-11-15T23:14:58,526 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T23:14:58,526 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-15T23:14:58,527 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-15T23:14:59,650 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-15T23:14:59,717 INFO [Time-limited test {}] log.Log(170): Logging initialized @2633ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-15T23:14:59,782 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T23:14:59,839 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T23:14:59,858 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T23:14:59,858 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T23:14:59,859 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T23:14:59,869 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T23:14:59,872 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/hadoop.log.dir/,AVAILABLE} 2024-11-15T23:14:59,873 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T23:15:00,046 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/java.io.tmpdir/jetty-localhost-40507-hadoop-hdfs-3_4_1-tests_jar-_-any-9508416817117805553/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T23:15:00,054 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:40507} 2024-11-15T23:15:00,055 INFO [Time-limited test {}] server.Server(415): Started @2971ms 2024-11-15T23:15:00,624 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T23:15:00,631 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T23:15:00,633 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T23:15:00,633 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T23:15:00,633 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T23:15:00,634 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/hadoop.log.dir/,AVAILABLE} 2024-11-15T23:15:00,634 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T23:15:00,732 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/java.io.tmpdir/jetty-localhost-43629-hadoop-hdfs-3_4_1-tests_jar-_-any-891077083555918500/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T23:15:00,733 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:43629} 2024-11-15T23:15:00,733 INFO [Time-limited test {}] server.Server(415): Started @3649ms 2024-11-15T23:15:00,779 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T23:15:01,815 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/cluster_535d8ff1-f3e7-a1c2-84cc-d10f4f48cd95/data/data1/current/BP-674942766-172.17.0.2-1731712499119/current, will proceed with Du for space computation calculation, 2024-11-15T23:15:01,815 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/cluster_535d8ff1-f3e7-a1c2-84cc-d10f4f48cd95/data/data2/current/BP-674942766-172.17.0.2-1731712499119/current, will proceed with Du for space computation calculation, 2024-11-15T23:15:01,845 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T23:15:01,890 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2dd6c4c25a3d738a with lease ID 0x60da42ce82c4a051: Processing first storage report for DS-294e9834-5e94-454d-916d-1d9f671474e7 from datanode DatanodeRegistration(127.0.0.1:37491, datanodeUuid=a6f950b1-7c3b-49b3-8987-cb2fd4e6642f, infoPort=39065, infoSecurePort=0, ipcPort=45319, storageInfo=lv=-57;cid=testClusterID;nsid=760348741;c=1731712499119) 2024-11-15T23:15:01,891 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2dd6c4c25a3d738a with lease ID 0x60da42ce82c4a051: from storage DS-294e9834-5e94-454d-916d-1d9f671474e7 node DatanodeRegistration(127.0.0.1:37491, datanodeUuid=a6f950b1-7c3b-49b3-8987-cb2fd4e6642f, infoPort=39065, infoSecurePort=0, ipcPort=45319, storageInfo=lv=-57;cid=testClusterID;nsid=760348741;c=1731712499119), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-15T23:15:01,892 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2dd6c4c25a3d738a with lease ID 0x60da42ce82c4a051: Processing first storage report for DS-3cc643f6-43f2-4e55-9126-be81ee15766e from datanode DatanodeRegistration(127.0.0.1:37491, datanodeUuid=a6f950b1-7c3b-49b3-8987-cb2fd4e6642f, infoPort=39065, infoSecurePort=0, ipcPort=45319, storageInfo=lv=-57;cid=testClusterID;nsid=760348741;c=1731712499119) 2024-11-15T23:15:01,892 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2dd6c4c25a3d738a with lease ID 0x60da42ce82c4a051: from storage DS-3cc643f6-43f2-4e55-9126-be81ee15766e node DatanodeRegistration(127.0.0.1:37491, datanodeUuid=a6f950b1-7c3b-49b3-8987-cb2fd4e6642f, infoPort=39065, infoSecurePort=0, ipcPort=45319, storageInfo=lv=-57;cid=testClusterID;nsid=760348741;c=1731712499119), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-15T23:15:01,972 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f 2024-11-15T23:15:02,043 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/cluster_535d8ff1-f3e7-a1c2-84cc-d10f4f48cd95/zookeeper_0, clientPort=65035, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/cluster_535d8ff1-f3e7-a1c2-84cc-d10f4f48cd95/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/cluster_535d8ff1-f3e7-a1c2-84cc-d10f4f48cd95/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T23:15:02,053 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=65035 2024-11-15T23:15:02,066 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T23:15:02,068 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T23:15:02,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741825_1001 (size=7) 2024-11-15T23:15:02,672 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444 with version=8 2024-11-15T23:15:02,673 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/hbase-staging 2024-11-15T23:15:02,762 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T23:15:03,012 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d6ecb9b7dda1:0 server-side Connection retries=45 2024-11-15T23:15:03,022 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T23:15:03,023 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T23:15:03,027 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T23:15:03,027 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T23:15:03,028 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T23:15:03,155 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T23:15:03,205 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T23:15:03,212 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T23:15:03,216 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:15:03,236 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 6590 (auto-detected) 2024-11-15T23:15:03,237 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-15T23:15:03,254 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43421 2024-11-15T23:15:03,274 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43421 connecting to ZooKeeper ensemble=127.0.0.1:65035 2024-11-15T23:15:03,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:434210x0, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T23:15:03,444 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43421-0x10140c7c4b40000 connected 2024-11-15T23:15:03,651 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T23:15:03,654 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T23:15:03,667 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T23:15:03,671 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444, hbase.cluster.distributed=false 2024-11-15T23:15:03,698 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T23:15:03,702 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43421 2024-11-15T23:15:03,703 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43421 2024-11-15T23:15:03,703 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43421 2024-11-15T23:15:03,704 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43421 2024-11-15T23:15:03,704 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43421 2024-11-15T23:15:03,793 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d6ecb9b7dda1:0 server-side Connection retries=45 2024-11-15T23:15:03,795 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T23:15:03,795 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T23:15:03,795 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T23:15:03,795 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T23:15:03,795 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T23:15:03,798 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T23:15:03,801 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:15:03,802 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38353 2024-11-15T23:15:03,804 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38353 connecting to ZooKeeper ensemble=127.0.0.1:65035 2024-11-15T23:15:03,806 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T23:15:03,810 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T23:15:03,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:383530x0, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T23:15:03,829 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38353-0x10140c7c4b40001 connected 2024-11-15T23:15:03,829 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T23:15:03,832 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T23:15:03,840 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T23:15:03,843 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T23:15:03,849 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T23:15:03,850 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38353 2024-11-15T23:15:03,851 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38353 2024-11-15T23:15:03,852 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38353 2024-11-15T23:15:03,854 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38353 2024-11-15T23:15:03,856 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38353 2024-11-15T23:15:03,871 DEBUG [M:0;d6ecb9b7dda1:43421 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d6ecb9b7dda1:43421 2024-11-15T23:15:03,872 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d6ecb9b7dda1,43421,1731712502855 2024-11-15T23:15:03,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T23:15:03,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T23:15:03,889 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d6ecb9b7dda1,43421,1731712502855 2024-11-15T23:15:03,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:15:03,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T23:15:03,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:15:03,920 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T23:15:03,921 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d6ecb9b7dda1,43421,1731712502855 from backup master directory 2024-11-15T23:15:03,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T23:15:03,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d6ecb9b7dda1,43421,1731712502855 2024-11-15T23:15:03,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T23:15:03,934 WARN [master/d6ecb9b7dda1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T23:15:03,934 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d6ecb9b7dda1,43421,1731712502855 2024-11-15T23:15:03,936 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-15T23:15:03,938 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-15T23:15:03,988 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/hbase.id] with ID: ddc94e31-6920-474d-af61-ead96ac60460 2024-11-15T23:15:03,989 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/.tmp/hbase.id 2024-11-15T23:15:03,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741826_1002 (size=42) 2024-11-15T23:15:04,404 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/.tmp/hbase.id]:[hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/hbase.id] 2024-11-15T23:15:04,459 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T23:15:04,464 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T23:15:04,483 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-15T23:15:04,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:15:04,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:15:04,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741827_1003 (size=196) 2024-11-15T23:15:04,986 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T23:15:04,989 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T23:15:05,007 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:05,012 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T23:15:05,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741828_1004 (size=1189) 2024-11-15T23:15:05,467 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store 2024-11-15T23:15:05,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741829_1005 (size=34) 2024-11-15T23:15:05,898 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-15T23:15:05,901 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:15:05,902 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T23:15:05,902 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T23:15:05,902 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T23:15:05,904 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T23:15:05,904 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T23:15:05,904 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T23:15:05,905 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731712505902Disabling compacts and flushes for region at 1731712505902Disabling writes for close at 1731712505904 (+2 ms)Writing region close event to WAL at 1731712505904Closed at 1731712505904 2024-11-15T23:15:05,908 WARN [master/d6ecb9b7dda1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store/.initializing 2024-11-15T23:15:05,908 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/WALs/d6ecb9b7dda1,43421,1731712502855 2024-11-15T23:15:05,916 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T23:15:05,929 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6ecb9b7dda1%2C43421%2C1731712502855, suffix=, logDir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/WALs/d6ecb9b7dda1,43421,1731712502855, archiveDir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/oldWALs, maxLogs=10 2024-11-15T23:15:05,949 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/WALs/d6ecb9b7dda1,43421,1731712502855/d6ecb9b7dda1%2C43421%2C1731712502855.1731712505933, exclude list is [], retry=0 2024-11-15T23:15:05,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37491,DS-294e9834-5e94-454d-916d-1d9f671474e7,DISK] 2024-11-15T23:15:05,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-15T23:15:05,998 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/WALs/d6ecb9b7dda1,43421,1731712502855/d6ecb9b7dda1%2C43421%2C1731712502855.1731712505933 2024-11-15T23:15:05,999 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39065:39065)] 2024-11-15T23:15:05,999 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T23:15:06,000 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:15:06,002 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:15:06,003 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:15:06,037 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:15:06,058 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T23:15:06,061 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:06,063 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T23:15:06,064 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:15:06,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T23:15:06,067 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:06,068 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:15:06,068 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:15:06,070 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T23:15:06,071 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:06,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:15:06,072 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:15:06,074 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T23:15:06,074 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:06,075 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:15:06,075 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:15:06,079 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:15:06,080 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:15:06,088 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:15:06,089 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:15:06,093 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T23:15:06,096 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:15:06,101 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T23:15:06,102 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73156619, jitterRate=0.09011857211589813}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T23:15:06,108 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731712506014Initializing all the Stores at 1731712506016 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731712506017 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712506017Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712506017Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712506017Cleaning up temporary data from old regions at 1731712506089 (+72 ms)Region opened successfully at 1731712506108 (+19 ms) 2024-11-15T23:15:06,110 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T23:15:06,142 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e10692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d6ecb9b7dda1/172.17.0.2:0 2024-11-15T23:15:06,169 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T23:15:06,179 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-15T23:15:06,179 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-15T23:15:06,182 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T23:15:06,183 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T23:15:06,188 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-15T23:15:06,188 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T23:15:06,213 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T23:15:06,222 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T23:15:06,291 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T23:15:06,296 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T23:15:06,298 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T23:15:06,308 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T23:15:06,310 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T23:15:06,315 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T23:15:06,322 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T23:15:06,324 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T23:15:06,333 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T23:15:06,353 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T23:15:06,364 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T23:15:06,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T23:15:06,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T23:15:06,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:15:06,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:15:06,379 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d6ecb9b7dda1,43421,1731712502855, sessionid=0x10140c7c4b40000, setting cluster-up flag (Was=false) 2024-11-15T23:15:06,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:15:06,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:15:06,449 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T23:15:06,455 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d6ecb9b7dda1,43421,1731712502855 2024-11-15T23:15:06,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:15:06,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:15:06,512 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T23:15:06,515 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d6ecb9b7dda1,43421,1731712502855 2024-11-15T23:15:06,525 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T23:15:06,563 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(746): ClusterId : ddc94e31-6920-474d-af61-ead96ac60460 2024-11-15T23:15:06,566 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T23:15:06,577 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T23:15:06,577 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T23:15:06,587 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T23:15:06,588 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@224ab3f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d6ecb9b7dda1/172.17.0.2:0 2024-11-15T23:15:06,596 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T23:15:06,605 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d6ecb9b7dda1:38353 2024-11-15T23:15:06,605 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T23:15:06,608 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T23:15:06,608 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T23:15:06,609 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T23:15:06,611 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(2659): reportForDuty to master=d6ecb9b7dda1,43421,1731712502855 with port=38353, startcode=1731712503763 2024-11-15T23:15:06,611 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T23:15:06,617 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d6ecb9b7dda1,43421,1731712502855 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T23:15:06,622 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T23:15:06,623 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d6ecb9b7dda1:0, corePoolSize=5, maxPoolSize=5 2024-11-15T23:15:06,623 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d6ecb9b7dda1:0, corePoolSize=5, maxPoolSize=5 2024-11-15T23:15:06,623 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d6ecb9b7dda1:0, corePoolSize=5, maxPoolSize=5 2024-11-15T23:15:06,623 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d6ecb9b7dda1:0, corePoolSize=5, maxPoolSize=5 2024-11-15T23:15:06,623 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d6ecb9b7dda1:0, corePoolSize=10, maxPoolSize=10 2024-11-15T23:15:06,624 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d6ecb9b7dda1:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:15:06,624 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d6ecb9b7dda1:0, corePoolSize=2, maxPoolSize=2 2024-11-15T23:15:06,624 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d6ecb9b7dda1:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:15:06,625 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731712536625 2024-11-15T23:15:06,627 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T23:15:06,628 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T23:15:06,629 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T23:15:06,630 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T23:15:06,631 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T23:15:06,631 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T23:15:06,632 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T23:15:06,632 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T23:15:06,633 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:06,635 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:06,636 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T23:15:06,635 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T23:15:06,637 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T23:15:06,638 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T23:15:06,641 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T23:15:06,641 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T23:15:06,644 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d6ecb9b7dda1:0:becomeActiveMaster-HFileCleaner.large.0-1731712506643,5,FailOnTimeoutGroup] 2024-11-15T23:15:06,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741831_1007 (size=1321) 2024-11-15T23:15:06,647 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d6ecb9b7dda1:0:becomeActiveMaster-HFileCleaner.small.0-1731712506645,5,FailOnTimeoutGroup] 2024-11-15T23:15:06,647 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:06,648 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T23:15:06,650 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:06,651 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:06,659 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42169, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T23:15:06,666 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43421 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:06,669 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43421 {}] master.ServerManager(517): Registering regionserver=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:06,682 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444 2024-11-15T23:15:06,683 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34017 2024-11-15T23:15:06,683 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T23:15:06,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T23:15:06,698 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] zookeeper.ZKUtil(111): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:06,698 WARN [RS:0;d6ecb9b7dda1:38353 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T23:15:06,698 INFO [RS:0;d6ecb9b7dda1:38353 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T23:15:06,699 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/WALs/d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:06,700 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d6ecb9b7dda1,38353,1731712503763] 2024-11-15T23:15:06,723 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T23:15:06,739 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T23:15:06,743 INFO [RS:0;d6ecb9b7dda1:38353 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T23:15:06,743 INFO [RS:0;d6ecb9b7dda1:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:06,744 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T23:15:06,749 INFO [RS:0;d6ecb9b7dda1:38353 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T23:15:06,750 INFO [RS:0;d6ecb9b7dda1:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:06,750 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:15:06,750 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d6ecb9b7dda1:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:15:06,750 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d6ecb9b7dda1:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:15:06,751 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:15:06,751 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d6ecb9b7dda1:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:15:06,751 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d6ecb9b7dda1:0, corePoolSize=2, maxPoolSize=2 2024-11-15T23:15:06,751 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:15:06,751 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d6ecb9b7dda1:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:15:06,751 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d6ecb9b7dda1:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:15:06,751 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d6ecb9b7dda1:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:15:06,751 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d6ecb9b7dda1:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:15:06,752 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d6ecb9b7dda1:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:15:06,752 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d6ecb9b7dda1:0, corePoolSize=3, maxPoolSize=3 2024-11-15T23:15:06,752 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0, corePoolSize=3, maxPoolSize=3 2024-11-15T23:15:06,753 INFO [RS:0;d6ecb9b7dda1:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:06,753 INFO [RS:0;d6ecb9b7dda1:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:06,753 INFO [RS:0;d6ecb9b7dda1:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:06,753 INFO [RS:0;d6ecb9b7dda1:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:06,753 INFO [RS:0;d6ecb9b7dda1:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:06,753 INFO [RS:0;d6ecb9b7dda1:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6ecb9b7dda1,38353,1731712503763-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T23:15:06,770 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T23:15:06,772 INFO [RS:0;d6ecb9b7dda1:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6ecb9b7dda1,38353,1731712503763-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:06,772 INFO [RS:0;d6ecb9b7dda1:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:06,772 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.Replication(171): d6ecb9b7dda1,38353,1731712503763 started 2024-11-15T23:15:06,789 INFO [RS:0;d6ecb9b7dda1:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:06,789 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(1482): Serving as d6ecb9b7dda1,38353,1731712503763, RpcServer on d6ecb9b7dda1/172.17.0.2:38353, sessionid=0x10140c7c4b40001 2024-11-15T23:15:06,790 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T23:15:06,790 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:06,790 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6ecb9b7dda1,38353,1731712503763' 2024-11-15T23:15:06,790 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T23:15:06,791 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T23:15:06,792 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T23:15:06,792 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T23:15:06,792 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:06,792 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6ecb9b7dda1,38353,1731712503763' 2024-11-15T23:15:06,792 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T23:15:06,793 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T23:15:06,794 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T23:15:06,794 INFO [RS:0;d6ecb9b7dda1:38353 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T23:15:06,794 INFO [RS:0;d6ecb9b7dda1:38353 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T23:15:06,900 INFO [RS:0;d6ecb9b7dda1:38353 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T23:15:06,905 INFO [RS:0;d6ecb9b7dda1:38353 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6ecb9b7dda1%2C38353%2C1731712503763, suffix=, logDir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/WALs/d6ecb9b7dda1,38353,1731712503763, archiveDir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/oldWALs, maxLogs=32 2024-11-15T23:15:06,921 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/WALs/d6ecb9b7dda1,38353,1731712503763/d6ecb9b7dda1%2C38353%2C1731712503763.1731712506909, exclude list is [], retry=0 2024-11-15T23:15:06,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37491,DS-294e9834-5e94-454d-916d-1d9f671474e7,DISK] 2024-11-15T23:15:06,929 INFO [RS:0;d6ecb9b7dda1:38353 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/WALs/d6ecb9b7dda1,38353,1731712503763/d6ecb9b7dda1%2C38353%2C1731712503763.1731712506909 2024-11-15T23:15:06,929 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39065:39065)] 2024-11-15T23:15:07,050 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T23:15:07,051 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444 2024-11-15T23:15:07,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741833_1009 (size=32) 2024-11-15T23:15:07,470 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:15:07,474 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T23:15:07,476 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T23:15:07,477 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:07,477 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T23:15:07,478 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T23:15:07,480 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T23:15:07,480 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:07,481 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T23:15:07,481 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T23:15:07,483 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T23:15:07,484 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:07,485 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T23:15:07,485 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T23:15:07,487 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T23:15:07,487 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:07,488 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T23:15:07,488 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T23:15:07,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740 2024-11-15T23:15:07,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740 2024-11-15T23:15:07,493 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T23:15:07,493 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T23:15:07,494 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T23:15:07,495 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T23:15:07,499 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T23:15:07,500 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66641985, jitterRate=-0.0069570392370224}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T23:15:07,503 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731712507470Initializing all the Stores at 1731712507473 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731712507473Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731712507474 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712507474Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731712507474Cleaning up temporary data from old regions at 1731712507493 (+19 ms)Region opened successfully at 1731712507503 (+10 ms) 2024-11-15T23:15:07,503 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T23:15:07,503 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T23:15:07,503 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T23:15:07,503 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T23:15:07,503 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T23:15:07,504 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T23:15:07,505 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731712507503Disabling compacts and flushes for region at 1731712507503Disabling writes for close at 1731712507503Writing region close event to WAL at 1731712507504 (+1 ms)Closed at 1731712507504 2024-11-15T23:15:07,508 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T23:15:07,508 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T23:15:07,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T23:15:07,520 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T23:15:07,523 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T23:15:07,680 DEBUG [d6ecb9b7dda1:43421 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-15T23:15:07,694 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:07,701 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d6ecb9b7dda1,38353,1731712503763, state=OPENING 2024-11-15T23:15:07,807 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T23:15:07,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:15:07,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:15:07,821 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T23:15:07,821 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T23:15:07,824 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T23:15:07,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d6ecb9b7dda1,38353,1731712503763}] 2024-11-15T23:15:08,008 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T23:15:08,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50767, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T23:15:08,021 INFO [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T23:15:08,022 INFO [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T23:15:08,023 INFO [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T23:15:08,026 INFO [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6ecb9b7dda1%2C38353%2C1731712503763.meta, suffix=.meta, logDir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/WALs/d6ecb9b7dda1,38353,1731712503763, archiveDir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/oldWALs, maxLogs=32 2024-11-15T23:15:08,041 DEBUG [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/WALs/d6ecb9b7dda1,38353,1731712503763/d6ecb9b7dda1%2C38353%2C1731712503763.meta.1731712508029.meta, exclude list is [], retry=0 2024-11-15T23:15:08,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37491,DS-294e9834-5e94-454d-916d-1d9f671474e7,DISK] 2024-11-15T23:15:08,048 INFO [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/WALs/d6ecb9b7dda1,38353,1731712503763/d6ecb9b7dda1%2C38353%2C1731712503763.meta.1731712508029.meta 2024-11-15T23:15:08,049 DEBUG [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39065:39065)] 2024-11-15T23:15:08,049 DEBUG [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T23:15:08,051 DEBUG [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T23:15:08,054 DEBUG [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T23:15:08,059 INFO [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T23:15:08,064 DEBUG [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T23:15:08,064 DEBUG [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:15:08,065 DEBUG [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T23:15:08,065 DEBUG [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T23:15:08,068 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T23:15:08,069 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T23:15:08,070 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:08,070 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T23:15:08,071 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T23:15:08,072 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T23:15:08,072 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:08,073 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T23:15:08,073 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T23:15:08,074 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T23:15:08,075 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:08,075 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T23:15:08,076 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T23:15:08,077 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T23:15:08,077 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:08,078 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T23:15:08,078 DEBUG [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T23:15:08,080 DEBUG [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740 2024-11-15T23:15:08,083 DEBUG [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740 2024-11-15T23:15:08,085 DEBUG [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T23:15:08,086 DEBUG [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T23:15:08,087 DEBUG [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T23:15:08,090 DEBUG [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T23:15:08,091 INFO [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71176143, jitterRate=0.0606071799993515}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T23:15:08,092 DEBUG [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T23:15:08,094 DEBUG [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731712508065Writing region info on filesystem at 1731712508066 (+1 ms)Initializing all the Stores at 1731712508067 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731712508067Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731712508068 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712508068Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731712508068Cleaning up temporary data from old regions at 1731712508086 (+18 ms)Running coprocessor post-open hooks at 1731712508092 (+6 ms)Region opened successfully at 1731712508094 (+2 ms) 2024-11-15T23:15:08,101 INFO [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731712507999 2024-11-15T23:15:08,113 DEBUG [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T23:15:08,113 INFO [RS_OPEN_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T23:15:08,115 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:08,117 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d6ecb9b7dda1,38353,1731712503763, state=OPEN 2024-11-15T23:15:08,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T23:15:08,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T23:15:08,153 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T23:15:08,153 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T23:15:08,153 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:08,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T23:15:08,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d6ecb9b7dda1,38353,1731712503763 in 325 msec 2024-11-15T23:15:08,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T23:15:08,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 647 msec 2024-11-15T23:15:08,167 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T23:15:08,167 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T23:15:08,185 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:08,187 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:08,208 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:08,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42587, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:08,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6810 sec 2024-11-15T23:15:08,234 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731712508234, completionTime=-1 2024-11-15T23:15:08,237 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-15T23:15:08,237 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T23:15:08,260 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-15T23:15:08,260 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731712568260 2024-11-15T23:15:08,260 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731712628260 2024-11-15T23:15:08,260 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 22 msec 2024-11-15T23:15:08,262 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6ecb9b7dda1,43421,1731712502855-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:08,262 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6ecb9b7dda1,43421,1731712502855-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:08,262 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6ecb9b7dda1,43421,1731712502855-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:08,264 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d6ecb9b7dda1:43421, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:08,264 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:08,264 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:08,269 DEBUG [master/d6ecb9b7dda1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T23:15:08,287 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.353sec 2024-11-15T23:15:08,288 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T23:15:08,289 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T23:15:08,290 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T23:15:08,291 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T23:15:08,291 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T23:15:08,291 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6ecb9b7dda1,43421,1731712502855-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T23:15:08,292 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6ecb9b7dda1,43421,1731712502855-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T23:15:08,301 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T23:15:08,302 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T23:15:08,302 INFO [master/d6ecb9b7dda1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6ecb9b7dda1,43421,1731712502855-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T23:15:08,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:08,378 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T23:15:08,379 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T23:15:08,381 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:08,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:08,390 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:08,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:08,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:08,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@576fe8ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:08,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:08,399 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:08,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:08,402 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:08,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:08,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:08,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d6ecb9b7dda1,43421,1731712502855 2024-11-15T23:15:08,443 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=353, ProcessCount=11, AvailableMemoryMB=6298 2024-11-15T23:15:08,451 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T23:15:08,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d6ecb9b7dda1,43421,1731712502855 2024-11-15T23:15:08,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b65baa9 2024-11-15T23:15:08,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T23:15:08,462 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T23:15:08,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43421 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T23:15:08,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43421 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T23:15:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T23:15:08,496 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T23:15:08,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43421 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-15T23:15:08,499 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:08,502 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T23:15:08,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T23:15:08,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741835_1011 (size=1061) 2024-11-15T23:15:08,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T23:15:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T23:15:08,920 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444 2024-11-15T23:15:08,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741836_1012 (size=53) 2024-11-15T23:15:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T23:15:09,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:15:09,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b9ac2b510c393ceae30043c6a9eaa588, disabling compactions & flushes 2024-11-15T23:15:09,333 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:09,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:09,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. after waiting 0 ms 2024-11-15T23:15:09,334 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:09,334 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:09,334 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b9ac2b510c393ceae30043c6a9eaa588: Waiting for close lock at 1731712509333Disabling compacts and flushes for region at 1731712509333Disabling writes for close at 1731712509333Writing region close event to WAL at 1731712509334 (+1 ms)Closed at 1731712509334 2024-11-15T23:15:09,338 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T23:15:09,345 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731712509338"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731712509338"}]},"ts":"1731712509338"} 2024-11-15T23:15:09,349 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T23:15:09,352 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T23:15:09,354 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712509352"}]},"ts":"1731712509352"} 2024-11-15T23:15:09,358 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T23:15:09,360 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9ac2b510c393ceae30043c6a9eaa588, ASSIGN}] 2024-11-15T23:15:09,364 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9ac2b510c393ceae30043c6a9eaa588, ASSIGN 2024-11-15T23:15:09,366 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9ac2b510c393ceae30043c6a9eaa588, ASSIGN; state=OFFLINE, location=d6ecb9b7dda1,38353,1731712503763; forceNewPlan=false, retain=false 2024-11-15T23:15:09,518 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b9ac2b510c393ceae30043c6a9eaa588, regionState=OPENING, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:09,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9ac2b510c393ceae30043c6a9eaa588, ASSIGN because future has completed 2024-11-15T23:15:09,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763}] 2024-11-15T23:15:09,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T23:15:09,690 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:09,690 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} 2024-11-15T23:15:09,691 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:09,691 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:15:09,691 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:09,691 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:09,694 INFO [StoreOpener-b9ac2b510c393ceae30043c6a9eaa588-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:09,697 INFO [StoreOpener-b9ac2b510c393ceae30043c6a9eaa588-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:15:09,697 INFO [StoreOpener-b9ac2b510c393ceae30043c6a9eaa588-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9ac2b510c393ceae30043c6a9eaa588 columnFamilyName A 2024-11-15T23:15:09,697 DEBUG [StoreOpener-b9ac2b510c393ceae30043c6a9eaa588-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:09,698 INFO [StoreOpener-b9ac2b510c393ceae30043c6a9eaa588-1 {}] regionserver.HStore(327): Store=b9ac2b510c393ceae30043c6a9eaa588/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:15:09,698 INFO [StoreOpener-b9ac2b510c393ceae30043c6a9eaa588-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:09,700 INFO [StoreOpener-b9ac2b510c393ceae30043c6a9eaa588-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:15:09,701 INFO [StoreOpener-b9ac2b510c393ceae30043c6a9eaa588-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9ac2b510c393ceae30043c6a9eaa588 columnFamilyName B 2024-11-15T23:15:09,701 DEBUG [StoreOpener-b9ac2b510c393ceae30043c6a9eaa588-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:09,702 INFO [StoreOpener-b9ac2b510c393ceae30043c6a9eaa588-1 {}] regionserver.HStore(327): Store=b9ac2b510c393ceae30043c6a9eaa588/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:15:09,702 INFO [StoreOpener-b9ac2b510c393ceae30043c6a9eaa588-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:09,704 INFO [StoreOpener-b9ac2b510c393ceae30043c6a9eaa588-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:15:09,704 INFO [StoreOpener-b9ac2b510c393ceae30043c6a9eaa588-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9ac2b510c393ceae30043c6a9eaa588 columnFamilyName C 2024-11-15T23:15:09,704 DEBUG [StoreOpener-b9ac2b510c393ceae30043c6a9eaa588-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:09,705 INFO [StoreOpener-b9ac2b510c393ceae30043c6a9eaa588-1 {}] regionserver.HStore(327): Store=b9ac2b510c393ceae30043c6a9eaa588/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:15:09,706 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:09,706 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:09,707 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:09,708 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:09,710 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:09,710 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:09,711 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T23:15:09,713 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:09,717 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T23:15:09,717 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b9ac2b510c393ceae30043c6a9eaa588; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73900450, jitterRate=0.10120251774787903}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T23:15:09,718 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:09,718 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b9ac2b510c393ceae30043c6a9eaa588: Running coprocessor pre-open hook at 1731712509691Writing region info on filesystem at 1731712509691Initializing all the Stores at 1731712509693 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712509693Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712509693Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712509693Cleaning up temporary data from old regions at 1731712509710 (+17 ms)Running coprocessor post-open hooks at 1731712509718 (+8 ms)Region opened successfully at 1731712509718 2024-11-15T23:15:09,720 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., pid=6, masterSystemTime=1731712509684 2024-11-15T23:15:09,724 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:09,724 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:09,725 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b9ac2b510c393ceae30043c6a9eaa588, regionState=OPEN, openSeqNum=2, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:09,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 because future has completed 2024-11-15T23:15:09,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-15T23:15:09,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 in 205 msec 2024-11-15T23:15:09,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-15T23:15:09,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9ac2b510c393ceae30043c6a9eaa588, ASSIGN in 375 msec 2024-11-15T23:15:09,741 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T23:15:09,741 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712509741"}]},"ts":"1731712509741"} 2024-11-15T23:15:09,745 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T23:15:09,747 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T23:15:09,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2760 sec 2024-11-15T23:15:10,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T23:15:10,669 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:10,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:10,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:10,684 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:10,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75100a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:10,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:10,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:10,687 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cfb3d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,689 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:10,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:10,696 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T23:15:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is d6ecb9b7dda1,43421,1731712502855 2024-11-15T23:15:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7bda4d25 2024-11-15T23:15:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T23:15:10,701 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T23:15:10,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,706 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:10,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:10,708 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:10,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:10,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ff6f12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:10,711 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:10,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bc6ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,713 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:10,713 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:10,714 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:10,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:10,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:10,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e153ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:10,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:10,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:10,718 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:10,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64257f01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,721 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:10,721 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:10,723 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:10,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:10,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ba4844, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:10,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:10,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:10,727 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:10,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c5b0754, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,729 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:10,729 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:10,730 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:10,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:10,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:10,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e855a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:10,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:10,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:10,733 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:10,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b24d866, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,735 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:10,736 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:10,737 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:10,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:10,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:10,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79ebe8ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:10,739 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c50a81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,742 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:10,742 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:10,743 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:10,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:10,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:10,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af5f6a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:10,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:10,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:10,747 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b504d21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,749 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:10,749 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:10,750 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3433011f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:10,752 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:10,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bcae178, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:10,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:10,756 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:10,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:10,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4686c825, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:10,758 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:10,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f254c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,761 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,761 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:10,761 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:10,762 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:10,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:10,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:10,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441ebf29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:10,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:10,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:10,764 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:10,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b6ade8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:10,769 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:10,769 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:10,769 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:10,769 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:10,769 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:10,769 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:10,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:10,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:10,773 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:10,773 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:10,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:10,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:10,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:10,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:10,777 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:10,778 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:10,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:10,779 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:10,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:10,781 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:10,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:10,782 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:10,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:15:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:15:10,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:10,785 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:15:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:15:10,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:15:10,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:15:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:10,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:10,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-15T23:15:10,797 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:15:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:15:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:15:10,799 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:15:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:15:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:15:10,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:15:10,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T23:15:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:10,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T23:15:10,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:10,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:10,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:10,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:10,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:10,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T23:15:10,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T23:15:11,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:11,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:11,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:11,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:11,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/86135e59455746a0b1669d265454d97a is 50, key is test_row_0/A:col10/1731712510857/Put/seqid=0 2024-11-15T23:15:11,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:11,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712570980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741837_1013 (size=19021) 2024-11-15T23:15:11,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/86135e59455746a0b1669d265454d97a 2024-11-15T23:15:11,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712570997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712571001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712571004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712570997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T23:15:11,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/b300bc1780d348a0836844d613c68d89 is 50, key is test_row_0/B:col10/1731712510857/Put/seqid=0 2024-11-15T23:15:11,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741838_1014 (size=12001) 2024-11-15T23:15:11,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T23:15:11,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712571177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712571178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712571177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:11,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712571178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:11,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:11,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:11,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:11,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712571187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T23:15:11,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:11,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:11,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:11,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:11,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712571399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712571401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712571401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712571402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712571410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T23:15:11,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T23:15:11,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:11,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:11,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:11,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:11,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:11,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/b300bc1780d348a0836844d613c68d89 2024-11-15T23:15:11,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/568c9f63dfcb4ad6bc7d1944e246124d is 50, key is test_row_0/C:col10/1731712510857/Put/seqid=0 2024-11-15T23:15:11,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741839_1015 (size=12001) 2024-11-15T23:15:11,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/568c9f63dfcb4ad6bc7d1944e246124d 2024-11-15T23:15:11,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T23:15:11,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:11,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:11,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:11,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:11,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:11,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:11,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/86135e59455746a0b1669d265454d97a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/86135e59455746a0b1669d265454d97a 2024-11-15T23:15:11,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/86135e59455746a0b1669d265454d97a, entries=300, sequenceid=13, filesize=18.6 K 2024-11-15T23:15:11,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/b300bc1780d348a0836844d613c68d89 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/b300bc1780d348a0836844d613c68d89 2024-11-15T23:15:11,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/b300bc1780d348a0836844d613c68d89, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T23:15:11,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/568c9f63dfcb4ad6bc7d1944e246124d as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/568c9f63dfcb4ad6bc7d1944e246124d 2024-11-15T23:15:11,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712571725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712571725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712571726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712571725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712571727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:11,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:11,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/568c9f63dfcb4ad6bc7d1944e246124d, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T23:15:11,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b9ac2b510c393ceae30043c6a9eaa588 in 898ms, sequenceid=13, compaction requested=false 2024-11-15T23:15:11,760 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T23:15:11,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:11,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T23:15:11,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:11,821 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T23:15:11,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:11,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:11,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:11,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:11,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:11,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:11,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/262c1611a77e47199ae8dac7e7e54790 is 50, key is test_row_0/A:col10/1731712510994/Put/seqid=0 2024-11-15T23:15:11,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741840_1016 (size=12001) 2024-11-15T23:15:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T23:15:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:12,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:12,254 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/262c1611a77e47199ae8dac7e7e54790 2024-11-15T23:15:12,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712572267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712572267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/6478d59005f548679e4f81c09bfbe74e is 50, key is test_row_0/B:col10/1731712510994/Put/seqid=0 2024-11-15T23:15:12,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712572272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712572273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712572283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741841_1017 (size=12001) 2024-11-15T23:15:12,308 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/6478d59005f548679e4f81c09bfbe74e 2024-11-15T23:15:12,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/66edd19b46614a6c82b7a6a1f980710a is 50, key is test_row_0/C:col10/1731712510994/Put/seqid=0 2024-11-15T23:15:12,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741842_1018 (size=12001) 2024-11-15T23:15:12,383 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/66edd19b46614a6c82b7a6a1f980710a 2024-11-15T23:15:12,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712572387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/262c1611a77e47199ae8dac7e7e54790 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/262c1611a77e47199ae8dac7e7e54790 2024-11-15T23:15:12,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712572398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712572398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712572398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712572399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,417 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/262c1611a77e47199ae8dac7e7e54790, entries=150, sequenceid=33, filesize=11.7 K 2024-11-15T23:15:12,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/6478d59005f548679e4f81c09bfbe74e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/6478d59005f548679e4f81c09bfbe74e 2024-11-15T23:15:12,440 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/6478d59005f548679e4f81c09bfbe74e, entries=150, sequenceid=33, filesize=11.7 K 2024-11-15T23:15:12,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/66edd19b46614a6c82b7a6a1f980710a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/66edd19b46614a6c82b7a6a1f980710a 2024-11-15T23:15:12,466 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/66edd19b46614a6c82b7a6a1f980710a, entries=150, sequenceid=33, filesize=11.7 K 2024-11-15T23:15:12,469 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b9ac2b510c393ceae30043c6a9eaa588 in 647ms, sequenceid=33, compaction requested=false 2024-11-15T23:15:12,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:12,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:12,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-15T23:15:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-15T23:15:12,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-15T23:15:12,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6710 sec 2024-11-15T23:15:12,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.6910 sec 2024-11-15T23:15:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:12,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T23:15:12,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:12,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:12,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:12,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:12,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:12,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:12,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/1337d2b9105c4081ab9b30df2e6d3270 is 50, key is test_row_0/A:col10/1731712512266/Put/seqid=0 2024-11-15T23:15:12,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741843_1019 (size=12001) 2024-11-15T23:15:12,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/1337d2b9105c4081ab9b30df2e6d3270 2024-11-15T23:15:12,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/4cc4c1ab4fff48dead1394b89efb1de3 is 50, key is test_row_0/B:col10/1731712512266/Put/seqid=0 2024-11-15T23:15:12,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712572670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712572674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712572679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712572683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712572686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,718 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T23:15:12,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741844_1020 (size=12001) 2024-11-15T23:15:12,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712572796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712572796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712572797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712572800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:12,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712572805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:12,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:12,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:12,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T23:15:12,956 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:12,957 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:15:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-15T23:15:12,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T23:15:12,961 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:15:12,963 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:15:12,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:15:13,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712573007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712573007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712573008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712573008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712573026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T23:15:13,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:15:13,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:13,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:13,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:13,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:13,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:13,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:13,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/4cc4c1ab4fff48dead1394b89efb1de3 2024-11-15T23:15:13,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/c5f44ae2a1774bd2a8b6fd7549573b03 is 50, key is test_row_0/C:col10/1731712512266/Put/seqid=0 2024-11-15T23:15:13,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741845_1021 (size=12001) 2024-11-15T23:15:13,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/c5f44ae2a1774bd2a8b6fd7549573b03 2024-11-15T23:15:13,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/1337d2b9105c4081ab9b30df2e6d3270 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/1337d2b9105c4081ab9b30df2e6d3270 2024-11-15T23:15:13,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/1337d2b9105c4081ab9b30df2e6d3270, entries=150, sequenceid=46, filesize=11.7 K 2024-11-15T23:15:13,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/4cc4c1ab4fff48dead1394b89efb1de3 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/4cc4c1ab4fff48dead1394b89efb1de3 2024-11-15T23:15:13,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/4cc4c1ab4fff48dead1394b89efb1de3, entries=150, sequenceid=46, filesize=11.7 K 2024-11-15T23:15:13,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/c5f44ae2a1774bd2a8b6fd7549573b03 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/c5f44ae2a1774bd2a8b6fd7549573b03 2024-11-15T23:15:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T23:15:13,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:15:13,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:13,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:13,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:13,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:13,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:13,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/c5f44ae2a1774bd2a8b6fd7549573b03, entries=150, sequenceid=46, filesize=11.7 K 2024-11-15T23:15:13,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b9ac2b510c393ceae30043c6a9eaa588 in 679ms, sequenceid=46, compaction requested=true 2024-11-15T23:15:13,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:13,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:15:13,295 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:13,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:13,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:15:13,295 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:13,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:13,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:15:13,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:13,300 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:13,302 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/B is initiating minor compaction (all files) 2024-11-15T23:15:13,303 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/B in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:13,303 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/b300bc1780d348a0836844d613c68d89, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/6478d59005f548679e4f81c09bfbe74e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/4cc4c1ab4fff48dead1394b89efb1de3] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=35.2 K 2024-11-15T23:15:13,305 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:13,305 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/A is initiating minor compaction (all files) 2024-11-15T23:15:13,305 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/A in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:13,305 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting b300bc1780d348a0836844d613c68d89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731712510844 2024-11-15T23:15:13,305 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/86135e59455746a0b1669d265454d97a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/262c1611a77e47199ae8dac7e7e54790, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/1337d2b9105c4081ab9b30df2e6d3270] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=42.0 K 2024-11-15T23:15:13,308 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86135e59455746a0b1669d265454d97a, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731712510797 2024-11-15T23:15:13,309 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 6478d59005f548679e4f81c09bfbe74e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731712510980 2024-11-15T23:15:13,310 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 262c1611a77e47199ae8dac7e7e54790, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731712510980 2024-11-15T23:15:13,310 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cc4c1ab4fff48dead1394b89efb1de3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731712512266 2024-11-15T23:15:13,312 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1337d2b9105c4081ab9b30df2e6d3270, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731712512266 2024-11-15T23:15:13,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:13,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T23:15:13,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:13,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:13,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:13,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:13,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:13,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:13,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/9ad5010c46e24b34afe4d0f78578e70e is 50, key is test_row_0/A:col10/1731712513316/Put/seqid=0 2024-11-15T23:15:13,369 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#A#compaction#10 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:13,370 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/3544a0a082884526a3c2adfcea6cc46d is 50, key is test_row_0/A:col10/1731712512266/Put/seqid=0 2024-11-15T23:15:13,371 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#B#compaction#11 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:13,372 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/3ec627692bd94b21a1236cf5bac34cb6 is 50, key is test_row_0/B:col10/1731712512266/Put/seqid=0 2024-11-15T23:15:13,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712573352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712573361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712573366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712573370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712573376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741846_1022 (size=14341) 2024-11-15T23:15:13,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741847_1023 (size=12104) 2024-11-15T23:15:13,428 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/3544a0a082884526a3c2adfcea6cc46d as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/3544a0a082884526a3c2adfcea6cc46d 2024-11-15T23:15:13,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741848_1024 (size=12104) 2024-11-15T23:15:13,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:15:13,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:13,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:13,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:13,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:13,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:13,452 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/3ec627692bd94b21a1236cf5bac34cb6 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/3ec627692bd94b21a1236cf5bac34cb6 2024-11-15T23:15:13,459 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/A of b9ac2b510c393ceae30043c6a9eaa588 into 3544a0a082884526a3c2adfcea6cc46d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:13,459 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:13,467 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/A, priority=13, startTime=1731712513291; duration=0sec 2024-11-15T23:15:13,468 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:13,468 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:A 2024-11-15T23:15:13,468 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:13,472 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:13,472 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/C is initiating minor compaction (all files) 2024-11-15T23:15:13,472 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/C in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:13,472 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/568c9f63dfcb4ad6bc7d1944e246124d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/66edd19b46614a6c82b7a6a1f980710a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/c5f44ae2a1774bd2a8b6fd7549573b03] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=35.2 K 2024-11-15T23:15:13,473 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 568c9f63dfcb4ad6bc7d1944e246124d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731712510844 2024-11-15T23:15:13,473 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/B of b9ac2b510c393ceae30043c6a9eaa588 into 3ec627692bd94b21a1236cf5bac34cb6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:13,474 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:13,474 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/B, priority=13, startTime=1731712513295; duration=0sec 2024-11-15T23:15:13,474 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66edd19b46614a6c82b7a6a1f980710a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731712510980 2024-11-15T23:15:13,474 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:13,474 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:B 2024-11-15T23:15:13,475 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5f44ae2a1774bd2a8b6fd7549573b03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731712512266 2024-11-15T23:15:13,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712573486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712573486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712573495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712573496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712573496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,519 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#C#compaction#12 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:13,524 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/798937d1ad244c71a1da5d64ef4544e3 is 50, key is test_row_0/C:col10/1731712512266/Put/seqid=0 2024-11-15T23:15:13,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741849_1025 (size=12104) 2024-11-15T23:15:13,556 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/798937d1ad244c71a1da5d64ef4544e3 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/798937d1ad244c71a1da5d64ef4544e3 2024-11-15T23:15:13,569 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/C of b9ac2b510c393ceae30043c6a9eaa588 into 798937d1ad244c71a1da5d64ef4544e3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:13,569 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:13,569 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/C, priority=13, startTime=1731712513295; duration=0sec 2024-11-15T23:15:13,570 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:13,570 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:C 2024-11-15T23:15:13,579 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T23:15:13,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T23:15:13,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:15:13,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:13,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:13,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:13,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:13,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:13,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:13,669 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T23:15:13,688 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-15T23:15:13,692 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T23:15:13,692 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-15T23:15:13,693 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-15T23:15:13,693 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-15T23:15:13,694 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T23:15:13,694 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T23:15:13,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712573696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712573705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712573706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712573706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712573707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:13,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:15:13,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:13,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:13,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:13,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:13,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:13,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:13,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/9ad5010c46e24b34afe4d0f78578e70e 2024-11-15T23:15:13,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/5f42f80aab144ccca99cdbdff10632bf is 50, key is test_row_0/B:col10/1731712513316/Put/seqid=0 2024-11-15T23:15:13,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741850_1026 (size=12001) 2024-11-15T23:15:13,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/5f42f80aab144ccca99cdbdff10632bf 2024-11-15T23:15:13,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:15:13,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:13,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:13,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:13,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:13,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:13,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:13,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/dd87e457751346908be1f58f4178aa3c is 50, key is test_row_0/C:col10/1731712513316/Put/seqid=0 2024-11-15T23:15:13,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741851_1027 (size=12001) 2024-11-15T23:15:14,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712574006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712574025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712574025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712574026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712574026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:15:14,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:14,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:14,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:14,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:14,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T23:15:14,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:15:14,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:14,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:14,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:14,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:14,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:14,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/dd87e457751346908be1f58f4178aa3c 2024-11-15T23:15:14,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/9ad5010c46e24b34afe4d0f78578e70e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/9ad5010c46e24b34afe4d0f78578e70e 2024-11-15T23:15:14,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:15:14,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:14,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:14,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:14,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:14,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:14,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/9ad5010c46e24b34afe4d0f78578e70e, entries=200, sequenceid=66, filesize=14.0 K 2024-11-15T23:15:14,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/5f42f80aab144ccca99cdbdff10632bf as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/5f42f80aab144ccca99cdbdff10632bf 2024-11-15T23:15:14,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/5f42f80aab144ccca99cdbdff10632bf, entries=150, sequenceid=66, filesize=11.7 K 2024-11-15T23:15:14,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/dd87e457751346908be1f58f4178aa3c as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/dd87e457751346908be1f58f4178aa3c 2024-11-15T23:15:14,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/dd87e457751346908be1f58f4178aa3c, entries=150, sequenceid=66, filesize=11.7 K 2024-11-15T23:15:14,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b9ac2b510c393ceae30043c6a9eaa588 in 1111ms, sequenceid=66, compaction requested=false 2024-11-15T23:15:14,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:14,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T23:15:14,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:14,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:14,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:14,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:14,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:14,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:14,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:15:14,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:14,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:14,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:14,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:14,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:14,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/5712a6cf5a1f43d1ab091fa936109ef5 is 50, key is test_row_0/A:col10/1731712513357/Put/seqid=0 2024-11-15T23:15:14,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712574569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712574570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741852_1028 (size=11997) 2024-11-15T23:15:14,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712574573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712574577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712574580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712574685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:15:14,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:14,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:14,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:14,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:14,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:14,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712574685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712574697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712574697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712574698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:15:14,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:14,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:14,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:14,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:14,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:14,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712574895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712574906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712574906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712574906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:14,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712574906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:14,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:14,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:14,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/5712a6cf5a1f43d1ab091fa936109ef5 2024-11-15T23:15:15,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:15:15,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:15,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:15,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:15,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:15,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:15,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/f079904b3c54460d883e3bac11b7d6d5 is 50, key is test_row_0/B:col10/1731712513357/Put/seqid=0 2024-11-15T23:15:15,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741853_1029 (size=9657) 2024-11-15T23:15:15,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/f079904b3c54460d883e3bac11b7d6d5 2024-11-15T23:15:15,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/0f36319ca2234e8688a4557634b3fd28 is 50, key is test_row_0/C:col10/1731712513357/Put/seqid=0 2024-11-15T23:15:15,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741854_1030 (size=9657) 2024-11-15T23:15:15,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/0f36319ca2234e8688a4557634b3fd28 2024-11-15T23:15:15,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/5712a6cf5a1f43d1ab091fa936109ef5 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/5712a6cf5a1f43d1ab091fa936109ef5 2024-11-15T23:15:15,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/5712a6cf5a1f43d1ab091fa936109ef5, entries=150, sequenceid=82, filesize=11.7 K 2024-11-15T23:15:15,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/f079904b3c54460d883e3bac11b7d6d5 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/f079904b3c54460d883e3bac11b7d6d5 2024-11-15T23:15:15,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/f079904b3c54460d883e3bac11b7d6d5, entries=100, sequenceid=82, filesize=9.4 K 2024-11-15T23:15:15,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/0f36319ca2234e8688a4557634b3fd28 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/0f36319ca2234e8688a4557634b3fd28 2024-11-15T23:15:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T23:15:15,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/0f36319ca2234e8688a4557634b3fd28, entries=100, sequenceid=82, filesize=9.4 K 2024-11-15T23:15:15,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b9ac2b510c393ceae30043c6a9eaa588 in 587ms, sequenceid=82, compaction requested=true 2024-11-15T23:15:15,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:15,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:15:15,116 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:15,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:15,117 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:15,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:15:15,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:15,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:15:15,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:15,118 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:15,118 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/B is initiating minor compaction (all files) 2024-11-15T23:15:15,119 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/B in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:15,119 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38442 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:15,119 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/A is initiating minor compaction (all files) 2024-11-15T23:15:15,119 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/3ec627692bd94b21a1236cf5bac34cb6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/5f42f80aab144ccca99cdbdff10632bf, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/f079904b3c54460d883e3bac11b7d6d5] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=33.0 K 2024-11-15T23:15:15,119 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/A in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:15,119 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/3544a0a082884526a3c2adfcea6cc46d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/9ad5010c46e24b34afe4d0f78578e70e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/5712a6cf5a1f43d1ab091fa936109ef5] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=37.5 K 2024-11-15T23:15:15,120 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3544a0a082884526a3c2adfcea6cc46d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731712512266 2024-11-15T23:15:15,120 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ec627692bd94b21a1236cf5bac34cb6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731712512266 2024-11-15T23:15:15,121 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f42f80aab144ccca99cdbdff10632bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731712512668 2024-11-15T23:15:15,122 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ad5010c46e24b34afe4d0f78578e70e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731712512668 2024-11-15T23:15:15,123 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting f079904b3c54460d883e3bac11b7d6d5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731712513357 2024-11-15T23:15:15,125 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5712a6cf5a1f43d1ab091fa936109ef5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731712513357 2024-11-15T23:15:15,144 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:15,146 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/57e4f44c4a9443a0b8781a7eed2dfc2f is 50, key is test_row_0/A:col10/1731712513357/Put/seqid=0 2024-11-15T23:15:15,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:15:15,165 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#B#compaction#19 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:15,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:15,166 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T23:15:15,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:15,166 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/ec6b6ec26a514ab089e78c0aa1924f5b is 50, key is test_row_0/B:col10/1731712513357/Put/seqid=0 2024-11-15T23:15:15,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:15,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:15,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:15,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:15,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:15,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741856_1032 (size=12207) 2024-11-15T23:15:15,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/6746ccf2504a4fceaacdb04819732e80 is 50, key is test_row_0/A:col10/1731712514565/Put/seqid=0 2024-11-15T23:15:15,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:15,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741855_1031 (size=12207) 2024-11-15T23:15:15,221 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/ec6b6ec26a514ab089e78c0aa1924f5b as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/ec6b6ec26a514ab089e78c0aa1924f5b 2024-11-15T23:15:15,226 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/57e4f44c4a9443a0b8781a7eed2dfc2f as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/57e4f44c4a9443a0b8781a7eed2dfc2f 2024-11-15T23:15:15,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741857_1033 (size=12001) 2024-11-15T23:15:15,243 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/B of b9ac2b510c393ceae30043c6a9eaa588 into ec6b6ec26a514ab089e78c0aa1924f5b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:15,243 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/6746ccf2504a4fceaacdb04819732e80 2024-11-15T23:15:15,244 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:15,244 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/B, priority=13, startTime=1731712515117; duration=0sec 2024-11-15T23:15:15,245 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/A of b9ac2b510c393ceae30043c6a9eaa588 into 57e4f44c4a9443a0b8781a7eed2dfc2f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:15,245 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:15,245 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:15,245 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:B 2024-11-15T23:15:15,245 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/A, priority=13, startTime=1731712515116; duration=0sec 2024-11-15T23:15:15,245 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:15,246 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:15,246 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:A 2024-11-15T23:15:15,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712575235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:15,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712575238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712575239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:15,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,252 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712575241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,252 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/C is initiating minor compaction (all files) 2024-11-15T23:15:15,253 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/C in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,253 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/798937d1ad244c71a1da5d64ef4544e3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/dd87e457751346908be1f58f4178aa3c, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/0f36319ca2234e8688a4557634b3fd28] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=33.0 K 2024-11-15T23:15:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712575243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:15,254 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 798937d1ad244c71a1da5d64ef4544e3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731712512266 2024-11-15T23:15:15,256 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting dd87e457751346908be1f58f4178aa3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731712512668 2024-11-15T23:15:15,257 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f36319ca2234e8688a4557634b3fd28, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731712513357 2024-11-15T23:15:15,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/f099ee56e1804087b04114a764029da8 is 50, key is test_row_0/B:col10/1731712514565/Put/seqid=0 2024-11-15T23:15:15,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741858_1034 (size=12001) 2024-11-15T23:15:15,303 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/f099ee56e1804087b04114a764029da8 2024-11-15T23:15:15,305 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#C#compaction#22 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:15,306 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/9d161af91aa0478f8955497b546d7724 is 50, key is test_row_0/C:col10/1731712513357/Put/seqid=0 2024-11-15T23:15:15,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/dfbeb81f559c439b9500c670bd9f3652 is 50, key is test_row_0/C:col10/1731712514565/Put/seqid=0 2024-11-15T23:15:15,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741859_1035 (size=12207) 2024-11-15T23:15:15,345 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/9d161af91aa0478f8955497b546d7724 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/9d161af91aa0478f8955497b546d7724 2024-11-15T23:15:15,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712575356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:15,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741860_1036 (size=12001) 2024-11-15T23:15:15,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712575356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:15,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712575356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:15,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712575356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:15,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712575357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:15,363 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/C of b9ac2b510c393ceae30043c6a9eaa588 into 9d161af91aa0478f8955497b546d7724(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:15,363 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:15,363 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/C, priority=13, startTime=1731712515117; duration=0sec 2024-11-15T23:15:15,363 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:15,363 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:C 2024-11-15T23:15:15,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712575565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:15,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712575565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:15,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712575565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:15,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712575566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712575566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:15,761 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/dfbeb81f559c439b9500c670bd9f3652 2024-11-15T23:15:15,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/6746ccf2504a4fceaacdb04819732e80 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/6746ccf2504a4fceaacdb04819732e80 2024-11-15T23:15:15,781 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/6746ccf2504a4fceaacdb04819732e80, entries=150, sequenceid=101, filesize=11.7 K 2024-11-15T23:15:15,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/f099ee56e1804087b04114a764029da8 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/f099ee56e1804087b04114a764029da8 2024-11-15T23:15:15,798 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/f099ee56e1804087b04114a764029da8, entries=150, sequenceid=101, filesize=11.7 K 2024-11-15T23:15:15,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/dfbeb81f559c439b9500c670bd9f3652 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/dfbeb81f559c439b9500c670bd9f3652 2024-11-15T23:15:15,818 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/dfbeb81f559c439b9500c670bd9f3652, entries=150, sequenceid=101, filesize=11.7 K 2024-11-15T23:15:15,819 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for b9ac2b510c393ceae30043c6a9eaa588 in 654ms, sequenceid=101, compaction requested=false 2024-11-15T23:15:15,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:15,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:15,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-15T23:15:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-15T23:15:15,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-15T23:15:15,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8610 sec 2024-11-15T23:15:15,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.8760 sec 2024-11-15T23:15:15,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T23:15:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:15,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:15,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:15,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:15,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:15,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:15,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:15,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/0a92f1211fed440b8584ee2b472569af is 50, key is test_row_0/A:col10/1731712515239/Put/seqid=0 2024-11-15T23:15:15,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741861_1037 (size=14341) 2024-11-15T23:15:15,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712575909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:15,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712575920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712575920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712575922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712575922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:15,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:15,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:15,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:15,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:16,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:16,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712576025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:16,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:16,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712576035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712576036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:16,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:16,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712576036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:16,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712576036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:16,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:16,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:16,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:16,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712576235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:16,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:16,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712576245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:16,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:16,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712576245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:16,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712576246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:16,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712576246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:16,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:16,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/0a92f1211fed440b8584ee2b472569af 2024-11-15T23:15:16,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/993ed588f84d4677ade594156432db64 is 50, key is test_row_0/B:col10/1731712515239/Put/seqid=0 2024-11-15T23:15:16,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741862_1038 (size=12001) 2024-11-15T23:15:16,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712576546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:16,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:16,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712576556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:16,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:16,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712576556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:16,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712576556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:16,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712576556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:16,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/993ed588f84d4677ade594156432db64 2024-11-15T23:15:16,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/2082ec15859840ee84682d6a050f697a is 50, key is test_row_0/C:col10/1731712515239/Put/seqid=0 2024-11-15T23:15:16,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741863_1039 (size=12001) 2024-11-15T23:15:17,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712577066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:17,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712577066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:17,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712577066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:17,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712577066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:17,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:17,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:17,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:17,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712577077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T23:15:17,116 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:17,116 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:15:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-15T23:15:17,125 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:15:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T23:15:17,126 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:15:17,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:15:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T23:15:17,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/2082ec15859840ee84682d6a050f697a 2024-11-15T23:15:17,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/0a92f1211fed440b8584ee2b472569af as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/0a92f1211fed440b8584ee2b472569af 2024-11-15T23:15:17,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/0a92f1211fed440b8584ee2b472569af, entries=200, sequenceid=121, filesize=14.0 K 2024-11-15T23:15:17,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/993ed588f84d4677ade594156432db64 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/993ed588f84d4677ade594156432db64 2024-11-15T23:15:17,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/993ed588f84d4677ade594156432db64, entries=150, sequenceid=121, filesize=11.7 K 2024-11-15T23:15:17,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T23:15:17,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:17,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:17,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:17,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:17,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:17,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/2082ec15859840ee84682d6a050f697a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/2082ec15859840ee84682d6a050f697a 2024-11-15T23:15:17,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/2082ec15859840ee84682d6a050f697a, entries=150, sequenceid=121, filesize=11.7 K 2024-11-15T23:15:17,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for b9ac2b510c393ceae30043c6a9eaa588 in 1412ms, sequenceid=121, compaction requested=true 2024-11-15T23:15:17,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:17,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:15:17,296 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:17,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:17,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:15:17,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:17,296 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:17,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:15:17,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:17,298 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:17,298 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:17,298 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/A is initiating minor compaction (all files) 2024-11-15T23:15:17,298 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/B is initiating minor compaction (all files) 2024-11-15T23:15:17,298 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/A in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:17,298 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/B in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:17,298 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/ec6b6ec26a514ab089e78c0aa1924f5b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/f099ee56e1804087b04114a764029da8, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/993ed588f84d4677ade594156432db64] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=35.4 K 2024-11-15T23:15:17,298 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/57e4f44c4a9443a0b8781a7eed2dfc2f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/6746ccf2504a4fceaacdb04819732e80, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/0a92f1211fed440b8584ee2b472569af] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=37.6 K 2024-11-15T23:15:17,299 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57e4f44c4a9443a0b8781a7eed2dfc2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731712512668 2024-11-15T23:15:17,299 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting ec6b6ec26a514ab089e78c0aa1924f5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731712512668 2024-11-15T23:15:17,299 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6746ccf2504a4fceaacdb04819732e80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731712514565 2024-11-15T23:15:17,299 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting f099ee56e1804087b04114a764029da8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731712514565 2024-11-15T23:15:17,300 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 993ed588f84d4677ade594156432db64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731712515239 2024-11-15T23:15:17,300 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a92f1211fed440b8584ee2b472569af, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731712515223 2024-11-15T23:15:17,323 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#B#compaction#27 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:17,324 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/5147ecba55b943e0a4be48a9e20a3803 is 50, key is test_row_0/B:col10/1731712515239/Put/seqid=0 2024-11-15T23:15:17,324 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#A#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:17,325 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/e0e91254a90a46e6ae24a0b0230dec55 is 50, key is test_row_0/A:col10/1731712515239/Put/seqid=0 2024-11-15T23:15:17,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741864_1040 (size=12309) 2024-11-15T23:15:17,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741865_1041 (size=12309) 2024-11-15T23:15:17,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T23:15:17,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:17,436 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T23:15:17,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:17,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:17,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:17,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:17,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:17,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:17,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T23:15:17,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/8959b43e8e6f42fda6a25646d570cdc5 is 50, key is test_row_0/A:col10/1731712515901/Put/seqid=0 2024-11-15T23:15:17,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741866_1042 (size=12151) 2024-11-15T23:15:17,754 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/5147ecba55b943e0a4be48a9e20a3803 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/5147ecba55b943e0a4be48a9e20a3803 2024-11-15T23:15:17,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T23:15:17,771 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/B of b9ac2b510c393ceae30043c6a9eaa588 into 5147ecba55b943e0a4be48a9e20a3803(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:17,771 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/e0e91254a90a46e6ae24a0b0230dec55 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/e0e91254a90a46e6ae24a0b0230dec55 2024-11-15T23:15:17,771 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:17,771 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/B, priority=13, startTime=1731712517296; duration=0sec 2024-11-15T23:15:17,772 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:17,772 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:B 2024-11-15T23:15:17,772 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:17,777 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:17,778 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/C is initiating minor compaction (all files) 2024-11-15T23:15:17,778 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/C in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:17,778 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/9d161af91aa0478f8955497b546d7724, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/dfbeb81f559c439b9500c670bd9f3652, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/2082ec15859840ee84682d6a050f697a] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=35.4 K 2024-11-15T23:15:17,780 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d161af91aa0478f8955497b546d7724, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731712512668 2024-11-15T23:15:17,782 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting dfbeb81f559c439b9500c670bd9f3652, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731712514565 2024-11-15T23:15:17,783 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 2082ec15859840ee84682d6a050f697a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731712515239 2024-11-15T23:15:17,791 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/A of b9ac2b510c393ceae30043c6a9eaa588 into e0e91254a90a46e6ae24a0b0230dec55(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:17,791 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:17,791 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/A, priority=13, startTime=1731712517296; duration=0sec 2024-11-15T23:15:17,792 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:17,792 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:A 2024-11-15T23:15:17,803 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#C#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:17,804 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/0a746b4c350d45b09b7817cfc2be8803 is 50, key is test_row_0/C:col10/1731712515239/Put/seqid=0 2024-11-15T23:15:17,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741867_1043 (size=12309) 2024-11-15T23:15:17,873 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/8959b43e8e6f42fda6a25646d570cdc5 2024-11-15T23:15:17,874 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/0a746b4c350d45b09b7817cfc2be8803 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/0a746b4c350d45b09b7817cfc2be8803 2024-11-15T23:15:17,891 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/C of b9ac2b510c393ceae30043c6a9eaa588 into 0a746b4c350d45b09b7817cfc2be8803(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:17,891 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:17,891 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/C, priority=13, startTime=1731712517296; duration=0sec 2024-11-15T23:15:17,891 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:17,891 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:C 2024-11-15T23:15:17,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/e97e0d04df964965b31685ba7fdf7b0c is 50, key is test_row_0/B:col10/1731712515901/Put/seqid=0 2024-11-15T23:15:17,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741868_1044 (size=12151) 2024-11-15T23:15:17,963 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/e97e0d04df964965b31685ba7fdf7b0c 2024-11-15T23:15:17,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/db3a22c42b01488eaa305e499bf89df9 is 50, key is test_row_0/C:col10/1731712515901/Put/seqid=0 2024-11-15T23:15:18,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741869_1045 (size=12151) 2024-11-15T23:15:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:18,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:18,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712578129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712578128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712578189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712578189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712578189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T23:15:18,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712578295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712578296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712578305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712578305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712578305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,428 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/db3a22c42b01488eaa305e499bf89df9 2024-11-15T23:15:18,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/8959b43e8e6f42fda6a25646d570cdc5 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/8959b43e8e6f42fda6a25646d570cdc5 2024-11-15T23:15:18,455 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/8959b43e8e6f42fda6a25646d570cdc5, entries=150, sequenceid=136, filesize=11.9 K 2024-11-15T23:15:18,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/e97e0d04df964965b31685ba7fdf7b0c as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/e97e0d04df964965b31685ba7fdf7b0c 2024-11-15T23:15:18,467 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/e97e0d04df964965b31685ba7fdf7b0c, entries=150, sequenceid=136, filesize=11.9 K 2024-11-15T23:15:18,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/db3a22c42b01488eaa305e499bf89df9 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/db3a22c42b01488eaa305e499bf89df9 2024-11-15T23:15:18,490 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/db3a22c42b01488eaa305e499bf89df9, entries=150, sequenceid=136, filesize=11.9 K 2024-11-15T23:15:18,492 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b9ac2b510c393ceae30043c6a9eaa588 in 1055ms, sequenceid=136, compaction requested=false 2024-11-15T23:15:18,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:18,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:18,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-15T23:15:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-15T23:15:18,501 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-15T23:15:18,501 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3710 sec 2024-11-15T23:15:18,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.3870 sec 2024-11-15T23:15:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:18,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T23:15:18,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:18,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:18,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:18,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:18,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:18,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:18,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/2f8836115e644f93a8e9d264f324ea0b is 50, key is test_row_0/A:col10/1731712518510/Put/seqid=0 2024-11-15T23:15:18,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712578538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712578547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712578552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712578553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712578555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741870_1046 (size=14541) 2024-11-15T23:15:18,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712578666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712578675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712578676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712578676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712578676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712578876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712578885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712578885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712578885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:18,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712578885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:18,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:18,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:18,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/2f8836115e644f93a8e9d264f324ea0b 2024-11-15T23:15:18,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/d62a318ef1bd4c78a94c464ea5510ffe is 50, key is test_row_0/B:col10/1731712518510/Put/seqid=0 2024-11-15T23:15:19,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741871_1047 (size=12151) 2024-11-15T23:15:19,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712579185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:19,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712579195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:19,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712579196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:19,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712579196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:19,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712579196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:19,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T23:15:19,276 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:19,276 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:15:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-15T23:15:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T23:15:19,280 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:15:19,281 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:15:19,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:15:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T23:15:19,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/d62a318ef1bd4c78a94c464ea5510ffe 2024-11-15T23:15:19,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/6348b33c4f2d4598be2dd615fb424e19 is 50, key is test_row_0/C:col10/1731712518510/Put/seqid=0 2024-11-15T23:15:19,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741872_1048 (size=12151) 2024-11-15T23:15:19,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/6348b33c4f2d4598be2dd615fb424e19 2024-11-15T23:15:19,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T23:15:19,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:19,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:19,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:19,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:19,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:19,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/2f8836115e644f93a8e9d264f324ea0b as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/2f8836115e644f93a8e9d264f324ea0b 2024-11-15T23:15:19,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/2f8836115e644f93a8e9d264f324ea0b, entries=200, sequenceid=157, filesize=14.2 K 2024-11-15T23:15:19,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/d62a318ef1bd4c78a94c464ea5510ffe as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/d62a318ef1bd4c78a94c464ea5510ffe 2024-11-15T23:15:19,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/d62a318ef1bd4c78a94c464ea5510ffe, entries=150, sequenceid=157, filesize=11.9 K 2024-11-15T23:15:19,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/6348b33c4f2d4598be2dd615fb424e19 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/6348b33c4f2d4598be2dd615fb424e19 2024-11-15T23:15:19,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/6348b33c4f2d4598be2dd615fb424e19, entries=150, sequenceid=157, filesize=11.9 K 2024-11-15T23:15:19,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b9ac2b510c393ceae30043c6a9eaa588 in 963ms, sequenceid=157, compaction requested=true 2024-11-15T23:15:19,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:19,476 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:19,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:15:19,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:19,476 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:19,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:15:19,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:19,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:15:19,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:19,478 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:19,478 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:19,478 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/B is initiating minor compaction (all files) 2024-11-15T23:15:19,478 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/A is initiating minor compaction (all files) 2024-11-15T23:15:19,478 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/A in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:19,478 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/B in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:19,478 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/e0e91254a90a46e6ae24a0b0230dec55, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/8959b43e8e6f42fda6a25646d570cdc5, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/2f8836115e644f93a8e9d264f324ea0b] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=38.1 K 2024-11-15T23:15:19,478 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/5147ecba55b943e0a4be48a9e20a3803, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/e97e0d04df964965b31685ba7fdf7b0c, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/d62a318ef1bd4c78a94c464ea5510ffe] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=35.8 K 2024-11-15T23:15:19,479 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0e91254a90a46e6ae24a0b0230dec55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731712515239 2024-11-15T23:15:19,479 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 5147ecba55b943e0a4be48a9e20a3803, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731712515239 2024-11-15T23:15:19,479 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8959b43e8e6f42fda6a25646d570cdc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731712515901 2024-11-15T23:15:19,479 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting e97e0d04df964965b31685ba7fdf7b0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731712515901 2024-11-15T23:15:19,480 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f8836115e644f93a8e9d264f324ea0b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731712518129 2024-11-15T23:15:19,480 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting d62a318ef1bd4c78a94c464ea5510ffe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731712518135 2024-11-15T23:15:19,491 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:19,491 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:19,491 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/9354c1512dcf44ef8f77d6bbc4f98b1b is 50, key is test_row_0/B:col10/1731712518510/Put/seqid=0 2024-11-15T23:15:19,491 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/338024f06c4342dbac1a68c61580fa49 is 50, key is test_row_0/A:col10/1731712518510/Put/seqid=0 2024-11-15T23:15:19,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741873_1049 (size=12561) 2024-11-15T23:15:19,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741874_1050 (size=12561) 2024-11-15T23:15:19,508 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/9354c1512dcf44ef8f77d6bbc4f98b1b as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/9354c1512dcf44ef8f77d6bbc4f98b1b 2024-11-15T23:15:19,519 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/B of b9ac2b510c393ceae30043c6a9eaa588 into 9354c1512dcf44ef8f77d6bbc4f98b1b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:19,519 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:19,519 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/B, priority=13, startTime=1731712519476; duration=0sec 2024-11-15T23:15:19,520 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:19,520 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:B 2024-11-15T23:15:19,520 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:19,521 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:19,521 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/C is initiating minor compaction (all files) 2024-11-15T23:15:19,521 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/C in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:19,522 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/0a746b4c350d45b09b7817cfc2be8803, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/db3a22c42b01488eaa305e499bf89df9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/6348b33c4f2d4598be2dd615fb424e19] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=35.8 K 2024-11-15T23:15:19,522 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a746b4c350d45b09b7817cfc2be8803, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731712515239 2024-11-15T23:15:19,523 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting db3a22c42b01488eaa305e499bf89df9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731712515901 2024-11-15T23:15:19,523 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 6348b33c4f2d4598be2dd615fb424e19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731712518135 2024-11-15T23:15:19,542 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#C#compaction#38 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:19,543 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/a1e376d914e24483a9c07f3677f5e6d0 is 50, key is test_row_0/C:col10/1731712518510/Put/seqid=0 2024-11-15T23:15:19,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741875_1051 (size=12561) 2024-11-15T23:15:19,562 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/a1e376d914e24483a9c07f3677f5e6d0 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/a1e376d914e24483a9c07f3677f5e6d0 2024-11-15T23:15:19,571 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/C of b9ac2b510c393ceae30043c6a9eaa588 into a1e376d914e24483a9c07f3677f5e6d0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:19,571 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:19,571 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/C, priority=13, startTime=1731712519476; duration=0sec 2024-11-15T23:15:19,572 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:19,572 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:C 2024-11-15T23:15:19,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T23:15:19,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:19,591 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T23:15:19,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:19,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:19,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:19,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:19,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:19,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:19,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T23:15:19,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/1169f13a7ff74f048431a8d6430a058e is 50, key is test_row_0/A:col10/1731712518552/Put/seqid=0 2024-11-15T23:15:19,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741876_1052 (size=12151) 2024-11-15T23:15:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:19,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:19,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:19,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712579720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:19,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:19,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712579721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:19,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:19,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712579725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:19,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:19,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712579734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:19,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712579738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:19,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:19,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712579845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:19,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:19,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:19,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712579846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:19,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:19,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:19,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712579846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:19,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:19,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712579846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:19,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:19,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712579846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:19,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:19,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:19,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T23:15:19,913 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/338024f06c4342dbac1a68c61580fa49 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/338024f06c4342dbac1a68c61580fa49 2024-11-15T23:15:19,925 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/A of b9ac2b510c393ceae30043c6a9eaa588 into 338024f06c4342dbac1a68c61580fa49(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:19,925 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:19,926 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/A, priority=13, startTime=1731712519476; duration=0sec 2024-11-15T23:15:19,926 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:19,926 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:A 2024-11-15T23:15:20,016 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/1169f13a7ff74f048431a8d6430a058e 2024-11-15T23:15:20,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/58cefde13f7642d3ab7a6fa8c24498ae is 50, key is test_row_0/B:col10/1731712518552/Put/seqid=0 2024-11-15T23:15:20,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741877_1053 (size=12151) 2024-11-15T23:15:20,057 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/58cefde13f7642d3ab7a6fa8c24498ae 2024-11-15T23:15:20,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:20,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712580056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:20,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712580056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712580056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:20,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:20,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712580056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:20,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:20,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712580057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:20,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/789fbc5c8eb74d0dafb6f199a9a2045d is 50, key is test_row_0/C:col10/1731712518552/Put/seqid=0 2024-11-15T23:15:20,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741878_1054 (size=12151) 2024-11-15T23:15:20,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712580365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:20,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712580365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:20,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712580366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712580366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:20,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712580375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T23:15:20,493 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/789fbc5c8eb74d0dafb6f199a9a2045d 2024-11-15T23:15:20,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/1169f13a7ff74f048431a8d6430a058e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/1169f13a7ff74f048431a8d6430a058e 2024-11-15T23:15:20,508 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/1169f13a7ff74f048431a8d6430a058e, entries=150, sequenceid=173, filesize=11.9 K 2024-11-15T23:15:20,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/58cefde13f7642d3ab7a6fa8c24498ae as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/58cefde13f7642d3ab7a6fa8c24498ae 2024-11-15T23:15:20,517 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/58cefde13f7642d3ab7a6fa8c24498ae, entries=150, sequenceid=173, filesize=11.9 K 2024-11-15T23:15:20,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/789fbc5c8eb74d0dafb6f199a9a2045d as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/789fbc5c8eb74d0dafb6f199a9a2045d 2024-11-15T23:15:20,529 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/789fbc5c8eb74d0dafb6f199a9a2045d, entries=150, sequenceid=173, filesize=11.9 K 2024-11-15T23:15:20,530 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b9ac2b510c393ceae30043c6a9eaa588 in 940ms, sequenceid=173, compaction requested=false 2024-11-15T23:15:20,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:20,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:20,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-15T23:15:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-15T23:15:20,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-15T23:15:20,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2520 sec 2024-11-15T23:15:20,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.2660 sec 2024-11-15T23:15:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:20,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T23:15:20,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:20,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:20,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:20,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:20,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:20,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:20,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/f6f504a112814ba086a3498c1a4f8593 is 50, key is test_row_0/A:col10/1731712519731/Put/seqid=0 2024-11-15T23:15:20,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741879_1055 (size=12151) 2024-11-15T23:15:20,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712580896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712580896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712580897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:20,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712580897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712580898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:21,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712581006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:21,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:21,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712581007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712581007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:21,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712581007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:21,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:21,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712581007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:21,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:21,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:21,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712581215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:21,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712581215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:21,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712581215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712581215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:21,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712581215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:21,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:21,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/f6f504a112814ba086a3498c1a4f8593 2024-11-15T23:15:21,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/3a882c04e8f746a08e60a21df64b6f3d is 50, key is test_row_0/B:col10/1731712519731/Put/seqid=0 2024-11-15T23:15:21,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741880_1056 (size=12151) 2024-11-15T23:15:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T23:15:21,426 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:21,426 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:15:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-15T23:15:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T23:15:21,429 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:15:21,430 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:15:21,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:15:21,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712581525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:21,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712581526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:21,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:21,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712581526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712581526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712581526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T23:15:21,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T23:15:21,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:21,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:21,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:21,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:21,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:21,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/3a882c04e8f746a08e60a21df64b6f3d 2024-11-15T23:15:21,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/f65dbcf2a1824062ada229aa59b02531 is 50, key is test_row_0/C:col10/1731712519731/Put/seqid=0 2024-11-15T23:15:21,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T23:15:21,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:21,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:21,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:21,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:21,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:21,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741881_1057 (size=12151) 2024-11-15T23:15:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T23:15:21,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T23:15:21,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:21,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:21,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:21,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:21,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:22,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712582035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:22,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712582036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:22,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712582036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:22,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T23:15:22,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:22,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:22,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:22,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:22,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:22,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712582045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:22,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712582045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:22,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:22,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:22,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T23:15:22,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/f65dbcf2a1824062ada229aa59b02531 2024-11-15T23:15:22,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/f6f504a112814ba086a3498c1a4f8593 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/f6f504a112814ba086a3498c1a4f8593 2024-11-15T23:15:22,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/f6f504a112814ba086a3498c1a4f8593, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T23:15:22,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/3a882c04e8f746a08e60a21df64b6f3d as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/3a882c04e8f746a08e60a21df64b6f3d 2024-11-15T23:15:22,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/3a882c04e8f746a08e60a21df64b6f3d, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T23:15:22,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/f65dbcf2a1824062ada229aa59b02531 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/f65dbcf2a1824062ada229aa59b02531 2024-11-15T23:15:22,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/f65dbcf2a1824062ada229aa59b02531, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T23:15:22,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=73.80 KB/75570 for b9ac2b510c393ceae30043c6a9eaa588 in 1305ms, sequenceid=194, compaction requested=true 2024-11-15T23:15:22,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:15:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:22,184 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:15:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:22,184 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:15:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:22,185 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:22,185 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/A is initiating minor compaction (all files) 2024-11-15T23:15:22,185 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/A in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:22,186 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/338024f06c4342dbac1a68c61580fa49, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/1169f13a7ff74f048431a8d6430a058e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/f6f504a112814ba086a3498c1a4f8593] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=36.0 K 2024-11-15T23:15:22,186 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:22,186 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/B is initiating minor compaction (all files) 2024-11-15T23:15:22,186 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/B in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:22,186 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/9354c1512dcf44ef8f77d6bbc4f98b1b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/58cefde13f7642d3ab7a6fa8c24498ae, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/3a882c04e8f746a08e60a21df64b6f3d] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=36.0 K 2024-11-15T23:15:22,187 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 338024f06c4342dbac1a68c61580fa49, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731712518135 2024-11-15T23:15:22,187 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 9354c1512dcf44ef8f77d6bbc4f98b1b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731712518135 2024-11-15T23:15:22,187 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1169f13a7ff74f048431a8d6430a058e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731712518543 2024-11-15T23:15:22,187 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 58cefde13f7642d3ab7a6fa8c24498ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731712518543 2024-11-15T23:15:22,188 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6f504a112814ba086a3498c1a4f8593, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731712519716 2024-11-15T23:15:22,188 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a882c04e8f746a08e60a21df64b6f3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731712519716 2024-11-15T23:15:22,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T23:15:22,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:22,199 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T23:15:22,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:22,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:22,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:22,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:22,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:22,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:22,201 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#B#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:22,202 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/00fa81ef58e74b2fb26f45e0094916a9 is 50, key is test_row_0/B:col10/1731712519731/Put/seqid=0 2024-11-15T23:15:22,206 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#A#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:22,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/4f345c2ae5984108a3cd21382bf6889e is 50, key is test_row_0/A:col10/1731712520893/Put/seqid=0 2024-11-15T23:15:22,207 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/825b01e8e58d4475866d75d71ce65f1e is 50, key is test_row_0/A:col10/1731712519731/Put/seqid=0 2024-11-15T23:15:22,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741882_1058 (size=12663) 2024-11-15T23:15:22,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741883_1059 (size=12151) 2024-11-15T23:15:22,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741884_1060 (size=12663) 2024-11-15T23:15:22,244 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/825b01e8e58d4475866d75d71ce65f1e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/825b01e8e58d4475866d75d71ce65f1e 2024-11-15T23:15:22,252 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/A of b9ac2b510c393ceae30043c6a9eaa588 into 825b01e8e58d4475866d75d71ce65f1e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:22,252 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:22,252 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/A, priority=13, startTime=1731712522184; duration=0sec 2024-11-15T23:15:22,252 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:22,252 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:A 2024-11-15T23:15:22,252 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:22,253 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:22,254 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/C is initiating minor compaction (all files) 2024-11-15T23:15:22,254 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/C in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:22,254 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/a1e376d914e24483a9c07f3677f5e6d0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/789fbc5c8eb74d0dafb6f199a9a2045d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/f65dbcf2a1824062ada229aa59b02531] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=36.0 K 2024-11-15T23:15:22,254 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1e376d914e24483a9c07f3677f5e6d0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731712518135 2024-11-15T23:15:22,255 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 789fbc5c8eb74d0dafb6f199a9a2045d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731712518543 2024-11-15T23:15:22,255 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting f65dbcf2a1824062ada229aa59b02531, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731712519716 2024-11-15T23:15:22,265 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#C#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:22,266 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/54ec0ac2fb7140229df85e35cebda48e is 50, key is test_row_0/C:col10/1731712519731/Put/seqid=0 2024-11-15T23:15:22,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741885_1061 (size=12663) 2024-11-15T23:15:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T23:15:22,628 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/00fa81ef58e74b2fb26f45e0094916a9 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/00fa81ef58e74b2fb26f45e0094916a9 2024-11-15T23:15:22,632 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/4f345c2ae5984108a3cd21382bf6889e 2024-11-15T23:15:22,636 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/B of b9ac2b510c393ceae30043c6a9eaa588 into 00fa81ef58e74b2fb26f45e0094916a9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:22,636 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:22,636 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/B, priority=13, startTime=1731712522184; duration=0sec 2024-11-15T23:15:22,636 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:22,636 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:B 2024-11-15T23:15:22,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/e411a4018df144d182681119c67e4ac8 is 50, key is test_row_0/B:col10/1731712520893/Put/seqid=0 2024-11-15T23:15:22,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741886_1062 (size=12151) 2024-11-15T23:15:22,684 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/54ec0ac2fb7140229df85e35cebda48e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/54ec0ac2fb7140229df85e35cebda48e 2024-11-15T23:15:22,696 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/C of b9ac2b510c393ceae30043c6a9eaa588 into 54ec0ac2fb7140229df85e35cebda48e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:22,696 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:22,697 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/C, priority=13, startTime=1731712522184; duration=0sec 2024-11-15T23:15:22,697 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:22,697 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:C 2024-11-15T23:15:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:23,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:23,050 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/e411a4018df144d182681119c67e4ac8 2024-11-15T23:15:23,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/3172e9e9117e4568beb4e51c1bea1cc3 is 50, key is test_row_0/C:col10/1731712520893/Put/seqid=0 2024-11-15T23:15:23,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741887_1063 (size=12151) 2024-11-15T23:15:23,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712583076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712583079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712583081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712583081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712583109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712583185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712583195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712583195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712583195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712583216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712583396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712583406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712583406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712583406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712583426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,480 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/3172e9e9117e4568beb4e51c1bea1cc3 2024-11-15T23:15:23,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/4f345c2ae5984108a3cd21382bf6889e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/4f345c2ae5984108a3cd21382bf6889e 2024-11-15T23:15:23,495 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/4f345c2ae5984108a3cd21382bf6889e, entries=150, sequenceid=208, filesize=11.9 K 2024-11-15T23:15:23,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/e411a4018df144d182681119c67e4ac8 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/e411a4018df144d182681119c67e4ac8 2024-11-15T23:15:23,505 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/e411a4018df144d182681119c67e4ac8, entries=150, sequenceid=208, filesize=11.9 K 2024-11-15T23:15:23,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/3172e9e9117e4568beb4e51c1bea1cc3 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/3172e9e9117e4568beb4e51c1bea1cc3 2024-11-15T23:15:23,515 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/3172e9e9117e4568beb4e51c1bea1cc3, entries=150, sequenceid=208, filesize=11.9 K 2024-11-15T23:15:23,516 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for b9ac2b510c393ceae30043c6a9eaa588 in 1317ms, sequenceid=208, compaction requested=false 2024-11-15T23:15:23,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:23,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:23,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-15T23:15:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-15T23:15:23,523 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-15T23:15:23,523 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0890 sec 2024-11-15T23:15:23,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.0970 sec 2024-11-15T23:15:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T23:15:23,575 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:23,576 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:15:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-15T23:15:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T23:15:23,578 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:15:23,580 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:15:23,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:15:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T23:15:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:23,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T23:15:23,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:23,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:23,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:23,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:23,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:23,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:23,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/9df6b5d8da08460c824c70dbca699780 is 50, key is test_row_0/A:col10/1731712523706/Put/seqid=0 2024-11-15T23:15:23,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741888_1064 (size=14541) 2024-11-15T23:15:23,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/9df6b5d8da08460c824c70dbca699780 2024-11-15T23:15:23,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712583725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712583726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T23:15:23,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:23,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:23,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:23,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:23,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:23,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712583728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712583731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712583736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/5d3002d17ad04d1d9a31d169ef335e39 is 50, key is test_row_0/B:col10/1731712523706/Put/seqid=0 2024-11-15T23:15:23,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741889_1065 (size=12151) 2024-11-15T23:15:23,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/5d3002d17ad04d1d9a31d169ef335e39 2024-11-15T23:15:23,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/7ce04c434a95480e92b949596c314276 is 50, key is test_row_0/C:col10/1731712523706/Put/seqid=0 2024-11-15T23:15:23,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741890_1066 (size=12151) 2024-11-15T23:15:23,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/7ce04c434a95480e92b949596c314276 2024-11-15T23:15:23,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/9df6b5d8da08460c824c70dbca699780 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/9df6b5d8da08460c824c70dbca699780 2024-11-15T23:15:23,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/9df6b5d8da08460c824c70dbca699780, entries=200, sequenceid=231, filesize=14.2 K 2024-11-15T23:15:23,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/5d3002d17ad04d1d9a31d169ef335e39 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/5d3002d17ad04d1d9a31d169ef335e39 2024-11-15T23:15:23,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/5d3002d17ad04d1d9a31d169ef335e39, entries=150, sequenceid=231, filesize=11.9 K 2024-11-15T23:15:23,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/7ce04c434a95480e92b949596c314276 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/7ce04c434a95480e92b949596c314276 2024-11-15T23:15:23,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/7ce04c434a95480e92b949596c314276, entries=150, sequenceid=231, filesize=11.9 K 2024-11-15T23:15:23,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b9ac2b510c393ceae30043c6a9eaa588 in 113ms, sequenceid=231, compaction requested=true 2024-11-15T23:15:23,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:23,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:15:23,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:23,821 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:23,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:15:23,821 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:23,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:23,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:15:23,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:23,822 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:23,822 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/B is initiating minor compaction (all files) 2024-11-15T23:15:23,822 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/B in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:23,823 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/00fa81ef58e74b2fb26f45e0094916a9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/e411a4018df144d182681119c67e4ac8, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/5d3002d17ad04d1d9a31d169ef335e39] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=36.1 K 2024-11-15T23:15:23,823 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 00fa81ef58e74b2fb26f45e0094916a9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731712519716 2024-11-15T23:15:23,823 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting e411a4018df144d182681119c67e4ac8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731712520893 2024-11-15T23:15:23,824 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d3002d17ad04d1d9a31d169ef335e39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731712523075 2024-11-15T23:15:23,825 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:23,825 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/A is initiating minor compaction (all files) 2024-11-15T23:15:23,825 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/A in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:23,826 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/825b01e8e58d4475866d75d71ce65f1e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/4f345c2ae5984108a3cd21382bf6889e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/9df6b5d8da08460c824c70dbca699780] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=38.4 K 2024-11-15T23:15:23,826 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 825b01e8e58d4475866d75d71ce65f1e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731712519716 2024-11-15T23:15:23,827 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f345c2ae5984108a3cd21382bf6889e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731712520893 2024-11-15T23:15:23,828 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9df6b5d8da08460c824c70dbca699780, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731712523075 2024-11-15T23:15:23,842 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#B#compaction#54 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:23,843 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/3cc89c29471349b99da8a1adab9bec90 is 50, key is test_row_0/B:col10/1731712523706/Put/seqid=0 2024-11-15T23:15:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:23,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T23:15:23,843 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#A#compaction#55 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:23,844 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/8d11331ab19643abac0068be95ed4c2e is 50, key is test_row_0/A:col10/1731712523706/Put/seqid=0 2024-11-15T23:15:23,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:23,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:23,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:23,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:23,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:23,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:23,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/71c2aab306a740f48c06feb9014c83a1 is 50, key is test_row_0/A:col10/1731712523723/Put/seqid=0 2024-11-15T23:15:23,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741893_1069 (size=14541) 2024-11-15T23:15:23,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741891_1067 (size=12765) 2024-11-15T23:15:23,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712583878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712583878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712583883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T23:15:23,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:23,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:23,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:23,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:23,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712583886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T23:15:23,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741892_1068 (size=12765) 2024-11-15T23:15:23,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712583995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712583995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:23,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712583995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:24,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712584006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:24,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:24,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T23:15:24,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:24,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:24,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:24,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:24,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:24,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T23:15:24,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:24,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:24,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:24,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:24,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:24,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T23:15:24,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712584205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:24,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712584206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:24,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:24,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712584206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:24,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712584215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:24,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:24,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712584245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:24,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:24,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/71c2aab306a740f48c06feb9014c83a1 2024-11-15T23:15:24,280 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/3cc89c29471349b99da8a1adab9bec90 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/3cc89c29471349b99da8a1adab9bec90 2024-11-15T23:15:24,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/8edf663d8e764f5796b3c303c1437135 is 50, key is test_row_0/B:col10/1731712523723/Put/seqid=0 2024-11-15T23:15:24,292 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/B of b9ac2b510c393ceae30043c6a9eaa588 into 3cc89c29471349b99da8a1adab9bec90(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:24,292 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:24,292 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/B, priority=13, startTime=1731712523821; duration=0sec 2024-11-15T23:15:24,293 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:24,293 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:B 2024-11-15T23:15:24,293 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:24,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741894_1070 (size=12151) 2024-11-15T23:15:24,294 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:24,294 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/C is initiating minor compaction (all files) 2024-11-15T23:15:24,294 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/C in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:24,294 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/54ec0ac2fb7140229df85e35cebda48e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/3172e9e9117e4568beb4e51c1bea1cc3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/7ce04c434a95480e92b949596c314276] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=36.1 K 2024-11-15T23:15:24,295 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 54ec0ac2fb7140229df85e35cebda48e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731712519716 2024-11-15T23:15:24,295 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 3172e9e9117e4568beb4e51c1bea1cc3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731712520893 2024-11-15T23:15:24,295 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ce04c434a95480e92b949596c314276, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731712523075 2024-11-15T23:15:24,302 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#C#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:24,303 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/69c9410024954107b29e53e3f8cd9cfa is 50, key is test_row_0/C:col10/1731712523706/Put/seqid=0 2024-11-15T23:15:24,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741895_1071 (size=12765) 2024-11-15T23:15:24,336 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/8d11331ab19643abac0068be95ed4c2e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/8d11331ab19643abac0068be95ed4c2e 2024-11-15T23:15:24,342 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/A of b9ac2b510c393ceae30043c6a9eaa588 into 8d11331ab19643abac0068be95ed4c2e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:24,342 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:24,343 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/A, priority=13, startTime=1731712523821; duration=0sec 2024-11-15T23:15:24,343 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:24,343 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:A 2024-11-15T23:15:24,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T23:15:24,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:24,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:24,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:24,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:24,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:24,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T23:15:24,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:24,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:24,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712584515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:24,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:24,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712584516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:24,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:24,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:24,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712584516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:24,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:24,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712584525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:24,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T23:15:24,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:24,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:24,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:24,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:24,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:24,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/8edf663d8e764f5796b3c303c1437135 2024-11-15T23:15:24,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/fa4eec9e166542119aff1e0c011916bd is 50, key is test_row_0/C:col10/1731712523723/Put/seqid=0 2024-11-15T23:15:24,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741896_1072 (size=12151) 2024-11-15T23:15:24,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/fa4eec9e166542119aff1e0c011916bd 2024-11-15T23:15:24,714 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/69c9410024954107b29e53e3f8cd9cfa as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/69c9410024954107b29e53e3f8cd9cfa 2024-11-15T23:15:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T23:15:24,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/71c2aab306a740f48c06feb9014c83a1 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/71c2aab306a740f48c06feb9014c83a1 2024-11-15T23:15:24,721 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/C of b9ac2b510c393ceae30043c6a9eaa588 into 69c9410024954107b29e53e3f8cd9cfa(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:24,721 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:24,721 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/C, priority=13, startTime=1731712523821; duration=0sec 2024-11-15T23:15:24,721 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:24,722 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:C 2024-11-15T23:15:24,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/71c2aab306a740f48c06feb9014c83a1, entries=200, sequenceid=245, filesize=14.2 K 2024-11-15T23:15:24,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/8edf663d8e764f5796b3c303c1437135 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/8edf663d8e764f5796b3c303c1437135 2024-11-15T23:15:24,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/8edf663d8e764f5796b3c303c1437135, entries=150, sequenceid=245, filesize=11.9 K 2024-11-15T23:15:24,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/fa4eec9e166542119aff1e0c011916bd as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/fa4eec9e166542119aff1e0c011916bd 2024-11-15T23:15:24,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/fa4eec9e166542119aff1e0c011916bd, entries=150, sequenceid=245, filesize=11.9 K 2024-11-15T23:15:24,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b9ac2b510c393ceae30043c6a9eaa588 in 897ms, sequenceid=245, compaction requested=false 2024-11-15T23:15:24,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:24,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T23:15:24,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:24,811 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T23:15:24,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:24,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:24,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:24,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:24,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/c6c11f86e7f046ca82fb6bad49f2b35d is 50, key is test_row_0/A:col10/1731712523881/Put/seqid=0 2024-11-15T23:15:24,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741897_1073 (size=12301) 2024-11-15T23:15:24,821 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/c6c11f86e7f046ca82fb6bad49f2b35d 2024-11-15T23:15:24,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/6ec20158d3484247819dcb3b7532e82a is 50, key is test_row_0/B:col10/1731712523881/Put/seqid=0 2024-11-15T23:15:24,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741898_1074 (size=12301) 2024-11-15T23:15:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:25,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:25,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712585040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:25,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:25,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712585042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:25,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712585043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712585044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:25,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712585156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712585156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:25,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712585156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712585156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:25,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:25,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:25,234 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/6ec20158d3484247819dcb3b7532e82a 2024-11-15T23:15:25,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/a5b654005fe64a749c982ed270c5dd0d is 50, key is test_row_0/C:col10/1731712523881/Put/seqid=0 2024-11-15T23:15:25,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741899_1075 (size=12301) 2024-11-15T23:15:25,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:25,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712585265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:25,269 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_0' in b9ac2b510c393ceae30043c6a9eaa588 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:15:25,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:25,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:25,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712585365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:25,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712585365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712585365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:25,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712585366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:25,649 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/a5b654005fe64a749c982ed270c5dd0d 2024-11-15T23:15:25,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/c6c11f86e7f046ca82fb6bad49f2b35d as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/c6c11f86e7f046ca82fb6bad49f2b35d 2024-11-15T23:15:25,661 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/c6c11f86e7f046ca82fb6bad49f2b35d, entries=150, sequenceid=266, filesize=12.0 K 2024-11-15T23:15:25,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/6ec20158d3484247819dcb3b7532e82a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/6ec20158d3484247819dcb3b7532e82a 2024-11-15T23:15:25,668 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/6ec20158d3484247819dcb3b7532e82a, entries=150, sequenceid=266, filesize=12.0 K 2024-11-15T23:15:25,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/a5b654005fe64a749c982ed270c5dd0d as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/a5b654005fe64a749c982ed270c5dd0d 2024-11-15T23:15:25,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712585677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:25,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712585677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:25,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712585677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:25,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712585677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:25,682 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/a5b654005fe64a749c982ed270c5dd0d, entries=150, sequenceid=266, filesize=12.0 K 2024-11-15T23:15:25,684 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b9ac2b510c393ceae30043c6a9eaa588 in 872ms, sequenceid=266, compaction requested=true 2024-11-15T23:15:25,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:25,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:25,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-15T23:15:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-15T23:15:25,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-15T23:15:25,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1060 sec 2024-11-15T23:15:25,694 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.1160 sec 2024-11-15T23:15:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T23:15:25,726 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:25,727 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:15:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-15T23:15:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T23:15:25,730 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:15:25,731 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:15:25,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:15:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T23:15:25,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T23:15:25,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:25,892 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T23:15:25,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:25,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:25,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:25,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:25,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:25,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:25,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/592fdcdad13744909406446d92875d2d is 50, key is test_row_0/A:col10/1731712525041/Put/seqid=0 2024-11-15T23:15:25,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741900_1076 (size=12301) 2024-11-15T23:15:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T23:15:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:26,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:26,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712586221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:26,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712586228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712586229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:26,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712586229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:26,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:26,306 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/592fdcdad13744909406446d92875d2d 2024-11-15T23:15:26,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/2610b87b1e7f44e9bda2c89621d64e0b is 50, key is test_row_0/B:col10/1731712525041/Put/seqid=0 2024-11-15T23:15:26,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741901_1077 (size=12301) 2024-11-15T23:15:26,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712586336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:26,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:26,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712586336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:26,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712586336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712586336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T23:15:26,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712586545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:26,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:26,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712586545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:26,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712586545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:26,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712586545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:26,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:26,721 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/2610b87b1e7f44e9bda2c89621d64e0b 2024-11-15T23:15:26,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/55f8de73703f4e57928c69fd0b811bad is 50, key is test_row_0/C:col10/1731712525041/Put/seqid=0 2024-11-15T23:15:26,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741902_1078 (size=12301) 2024-11-15T23:15:26,735 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/55f8de73703f4e57928c69fd0b811bad 2024-11-15T23:15:26,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/592fdcdad13744909406446d92875d2d as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/592fdcdad13744909406446d92875d2d 2024-11-15T23:15:26,746 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/592fdcdad13744909406446d92875d2d, entries=150, sequenceid=280, filesize=12.0 K 2024-11-15T23:15:26,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/2610b87b1e7f44e9bda2c89621d64e0b as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/2610b87b1e7f44e9bda2c89621d64e0b 2024-11-15T23:15:26,753 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/2610b87b1e7f44e9bda2c89621d64e0b, entries=150, sequenceid=280, filesize=12.0 K 2024-11-15T23:15:26,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/55f8de73703f4e57928c69fd0b811bad as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/55f8de73703f4e57928c69fd0b811bad 2024-11-15T23:15:26,758 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/55f8de73703f4e57928c69fd0b811bad, entries=150, sequenceid=280, filesize=12.0 K 2024-11-15T23:15:26,760 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for b9ac2b510c393ceae30043c6a9eaa588 in 868ms, sequenceid=280, compaction requested=true 2024-11-15T23:15:26,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:26,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:26,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-15T23:15:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-15T23:15:26,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-15T23:15:26,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0300 sec 2024-11-15T23:15:26,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.0380 sec 2024-11-15T23:15:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:26,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T23:15:26,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T23:15:26,865 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:26,866 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:15:26,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:26,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:26,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:26,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:26,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-15T23:15:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T23:15:26,873 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:15:26,876 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:15:26,877 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:15:26,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/682e2bd73a014f8d99e27d2279433e00 is 50, key is test_row_0/A:col10/1731712526859/Put/seqid=0 2024-11-15T23:15:26,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712586885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:26,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:26,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741903_1079 (size=12301) 2024-11-15T23:15:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712586886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:26,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/682e2bd73a014f8d99e27d2279433e00 2024-11-15T23:15:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712586888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:26,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712586902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:26,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/76be7f812ab14a9997153a481bb6c2a3 is 50, key is test_row_0/B:col10/1731712526859/Put/seqid=0 2024-11-15T23:15:26,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741904_1080 (size=12301) 2024-11-15T23:15:26,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/76be7f812ab14a9997153a481bb6c2a3 2024-11-15T23:15:26,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/5f4bef66d77e440bb2c77069b810a1dc is 50, key is test_row_0/C:col10/1731712526859/Put/seqid=0 2024-11-15T23:15:26,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741905_1081 (size=12301) 2024-11-15T23:15:26,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/5f4bef66d77e440bb2c77069b810a1dc 2024-11-15T23:15:26,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/682e2bd73a014f8d99e27d2279433e00 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/682e2bd73a014f8d99e27d2279433e00 2024-11-15T23:15:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T23:15:26,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/682e2bd73a014f8d99e27d2279433e00, entries=150, sequenceid=301, filesize=12.0 K 2024-11-15T23:15:26,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/76be7f812ab14a9997153a481bb6c2a3 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/76be7f812ab14a9997153a481bb6c2a3 2024-11-15T23:15:26,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/76be7f812ab14a9997153a481bb6c2a3, entries=150, sequenceid=301, filesize=12.0 K 2024-11-15T23:15:26,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/5f4bef66d77e440bb2c77069b810a1dc as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/5f4bef66d77e440bb2c77069b810a1dc 2024-11-15T23:15:27,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/5f4bef66d77e440bb2c77069b810a1dc, entries=150, sequenceid=301, filesize=12.0 K 2024-11-15T23:15:27,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for b9ac2b510c393ceae30043c6a9eaa588 in 142ms, sequenceid=301, compaction requested=true 2024-11-15T23:15:27,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:27,004 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T23:15:27,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:15:27,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:27,005 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T23:15:27,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:15:27,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:27,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:15:27,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:27,006 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 64209 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T23:15:27,007 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/A is initiating minor compaction (all files) 2024-11-15T23:15:27,007 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/A in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:27,007 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/8d11331ab19643abac0068be95ed4c2e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/71c2aab306a740f48c06feb9014c83a1, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/c6c11f86e7f046ca82fb6bad49f2b35d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/592fdcdad13744909406446d92875d2d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/682e2bd73a014f8d99e27d2279433e00] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=62.7 K 2024-11-15T23:15:27,008 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d11331ab19643abac0068be95ed4c2e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731712523075 2024-11-15T23:15:27,008 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61819 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T23:15:27,008 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/B is initiating minor compaction (all files) 2024-11-15T23:15:27,008 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/B in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:27,008 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/3cc89c29471349b99da8a1adab9bec90, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/8edf663d8e764f5796b3c303c1437135, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/6ec20158d3484247819dcb3b7532e82a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/2610b87b1e7f44e9bda2c89621d64e0b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/76be7f812ab14a9997153a481bb6c2a3] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=60.4 K 2024-11-15T23:15:27,009 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cc89c29471349b99da8a1adab9bec90, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731712523075 2024-11-15T23:15:27,009 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71c2aab306a740f48c06feb9014c83a1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731712523720 2024-11-15T23:15:27,011 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 8edf663d8e764f5796b3c303c1437135, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731712523720 2024-11-15T23:15:27,012 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6c11f86e7f046ca82fb6bad49f2b35d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731712523875 2024-11-15T23:15:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:27,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T23:15:27,013 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 592fdcdad13744909406446d92875d2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731712525038 2024-11-15T23:15:27,013 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ec20158d3484247819dcb3b7532e82a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731712523875 2024-11-15T23:15:27,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:27,014 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 2610b87b1e7f44e9bda2c89621d64e0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731712525038 2024-11-15T23:15:27,014 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 682e2bd73a014f8d99e27d2279433e00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731712526227 2024-11-15T23:15:27,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:27,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:27,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:27,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:27,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:27,015 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 76be7f812ab14a9997153a481bb6c2a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731712526227 2024-11-15T23:15:27,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:15:27,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:27,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:27,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:27,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/6b1583e2ba4e4ead832691e829d0193d is 50, key is test_row_0/A:col10/1731712526898/Put/seqid=0 2024-11-15T23:15:27,049 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#B#compaction#70 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:27,049 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/35f1d60c4f3342c8bbd1302ffab88afd is 50, key is test_row_0/B:col10/1731712526859/Put/seqid=0 2024-11-15T23:15:27,060 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#A#compaction#71 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:27,061 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/a4387231906341ebb3e84b1a8a394945 is 50, key is test_row_0/A:col10/1731712526859/Put/seqid=0 2024-11-15T23:15:27,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741906_1082 (size=14741) 2024-11-15T23:15:27,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/6b1583e2ba4e4ead832691e829d0193d 2024-11-15T23:15:27,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741907_1083 (size=13085) 2024-11-15T23:15:27,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712587074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712587077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712527056, max timestamp=1731712527078Num uniques 859; 2024-11-15T23:15:27,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/193fcc3587964c958e28f10f7624a7b7 is 50, key is test_row_0/B:col10/1731712526898/Put/seqid=0 2024-11-15T23:15:27,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:15:27,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741908_1084 (size=13085) 2024-11-15T23:15:27,122 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/a4387231906341ebb3e84b1a8a394945 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/a4387231906341ebb3e84b1a8a394945 2024-11-15T23:15:27,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712527056, max timestamp=1731712527078Num uniques 859; 2024-11-15T23:15:27,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:15:27,136 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/A of b9ac2b510c393ceae30043c6a9eaa588 into a4387231906341ebb3e84b1a8a394945(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:27,136 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:27,136 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/A, priority=11, startTime=1731712527004; duration=0sec 2024-11-15T23:15:27,136 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:27,137 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:A 2024-11-15T23:15:27,137 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T23:15:27,138 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61819 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T23:15:27,139 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/C is initiating minor compaction (all files) 2024-11-15T23:15:27,139 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/C in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:27,139 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/69c9410024954107b29e53e3f8cd9cfa, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/fa4eec9e166542119aff1e0c011916bd, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/a5b654005fe64a749c982ed270c5dd0d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/55f8de73703f4e57928c69fd0b811bad, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/5f4bef66d77e440bb2c77069b810a1dc] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=60.4 K 2024-11-15T23:15:27,140 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69c9410024954107b29e53e3f8cd9cfa, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731712523075 2024-11-15T23:15:27,140 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa4eec9e166542119aff1e0c011916bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731712523720 2024-11-15T23:15:27,141 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5b654005fe64a749c982ed270c5dd0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731712523875 2024-11-15T23:15:27,141 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55f8de73703f4e57928c69fd0b811bad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731712525038 2024-11-15T23:15:27,144 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f4bef66d77e440bb2c77069b810a1dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731712526227 2024-11-15T23:15:27,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731712527056, max timestamp=1731712527108Num uniques 859; 2024-11-15T23:15:27,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T23:15:27,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741909_1085 (size=12301) 2024-11-15T23:15:27,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/193fcc3587964c958e28f10f7624a7b7 2024-11-15T23:15:27,180 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#C#compaction#73 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:27,181 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/53f1656164e64494be4ffb50e9ac8a7e is 50, key is test_row_0/C:col10/1731712526859/Put/seqid=0 2024-11-15T23:15:27,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:15:27,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T23:15:27,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/a16b1540936f49209823d28107269962 is 50, key is test_row_0/C:col10/1731712526898/Put/seqid=0 2024-11-15T23:15:27,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:27,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:27,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712587195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712587199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712587200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741910_1086 (size=13085) 2024-11-15T23:15:27,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741911_1087 (size=12301) 2024-11-15T23:15:27,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712587206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/a16b1540936f49209823d28107269962 2024-11-15T23:15:27,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/6b1583e2ba4e4ead832691e829d0193d as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/6b1583e2ba4e4ead832691e829d0193d 2024-11-15T23:15:27,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/6b1583e2ba4e4ead832691e829d0193d, entries=200, sequenceid=313, filesize=14.4 K 2024-11-15T23:15:27,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/193fcc3587964c958e28f10f7624a7b7 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/193fcc3587964c958e28f10f7624a7b7 2024-11-15T23:15:27,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/193fcc3587964c958e28f10f7624a7b7, entries=150, sequenceid=313, filesize=12.0 K 2024-11-15T23:15:27,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/a16b1540936f49209823d28107269962 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/a16b1540936f49209823d28107269962 2024-11-15T23:15:27,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/a16b1540936f49209823d28107269962, entries=150, sequenceid=313, filesize=12.0 K 2024-11-15T23:15:27,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.37 KB/123255 for b9ac2b510c393ceae30043c6a9eaa588 in 235ms, sequenceid=313, compaction requested=false 2024-11-15T23:15:27,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:27,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=127.08 KB heapSize=364.15 KB 2024-11-15T23:15:27,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:27,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:27,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:27,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:27,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:27,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:27,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/8f0bfee0383345028d3bfdf3ac617ed5 is 50, key is test_row_0/A:col10/1731712527283/Put/seqid=0 2024-11-15T23:15:27,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741912_1088 (size=14741) 2024-11-15T23:15:27,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/8f0bfee0383345028d3bfdf3ac617ed5 2024-11-15T23:15:27,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712587325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/d84ee5ef97ea42808d02a6cbc194e674 is 50, key is test_row_0/B:col10/1731712527283/Put/seqid=0 2024-11-15T23:15:27,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712587328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712587331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:15:27,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:27,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:27,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:27,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741913_1089 (size=12301) 2024-11-15T23:15:27,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712587405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712587425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712587435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712587445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712587445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,490 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/35f1d60c4f3342c8bbd1302ffab88afd as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/35f1d60c4f3342c8bbd1302ffab88afd 2024-11-15T23:15:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T23:15:27,496 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/B of b9ac2b510c393ceae30043c6a9eaa588 into 35f1d60c4f3342c8bbd1302ffab88afd(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:27,497 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:27,497 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/B, priority=11, startTime=1731712527005; duration=0sec 2024-11-15T23:15:27,497 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:27,497 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:B 2024-11-15T23:15:27,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:15:27,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:27,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:27,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:27,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,613 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/53f1656164e64494be4ffb50e9ac8a7e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/53f1656164e64494be4ffb50e9ac8a7e 2024-11-15T23:15:27,619 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/C of b9ac2b510c393ceae30043c6a9eaa588 into 53f1656164e64494be4ffb50e9ac8a7e(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:27,619 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:27,619 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/C, priority=11, startTime=1731712527006; duration=0sec 2024-11-15T23:15:27,619 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:27,619 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:C 2024-11-15T23:15:27,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:15:27,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:27,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:27,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:27,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712587655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712587655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712587655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712587725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712587735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/d84ee5ef97ea42808d02a6cbc194e674 2024-11-15T23:15:27,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/e0d6f3eb0449482aa72cf24b95378595 is 50, key is test_row_0/C:col10/1731712527283/Put/seqid=0 2024-11-15T23:15:27,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741914_1090 (size=12301) 2024-11-15T23:15:27,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:15:27,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:27,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:27,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:27,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:15:27,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:27,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:27,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:27,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:27,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712587965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712587975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:27,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712587975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:27,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:27,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:27,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T23:15:28,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:15:28,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:28,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:28,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:28,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:28,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:28,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/e0d6f3eb0449482aa72cf24b95378595 2024-11-15T23:15:28,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/8f0bfee0383345028d3bfdf3ac617ed5 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/8f0bfee0383345028d3bfdf3ac617ed5 2024-11-15T23:15:28,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/8f0bfee0383345028d3bfdf3ac617ed5, entries=200, sequenceid=350, filesize=14.4 K 2024-11-15T23:15:28,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/d84ee5ef97ea42808d02a6cbc194e674 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/d84ee5ef97ea42808d02a6cbc194e674 2024-11-15T23:15:28,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/d84ee5ef97ea42808d02a6cbc194e674, entries=150, sequenceid=350, filesize=12.0 K 2024-11-15T23:15:28,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/e0d6f3eb0449482aa72cf24b95378595 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/e0d6f3eb0449482aa72cf24b95378595 2024-11-15T23:15:28,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712588245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:28,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/e0d6f3eb0449482aa72cf24b95378595, entries=150, sequenceid=350, filesize=12.0 K 2024-11-15T23:15:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:28,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.08 KB/130125, heapSize ~364.10 KB/372840, currentSize=53.67 KB/54960 for b9ac2b510c393ceae30043c6a9eaa588 in 968ms, sequenceid=350, compaction requested=true 2024-11-15T23:15:28,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:28,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:15:28,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:28,252 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:28,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:15:28,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:28,252 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:28,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:15:28,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:28,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T23:15:28,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:28,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:28,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:28,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:28,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:28,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:28,255 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42567 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:28,255 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:28,255 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/A is initiating minor compaction (all files) 2024-11-15T23:15:28,255 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/B is initiating minor compaction (all files) 2024-11-15T23:15:28,255 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/A in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:28,255 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/B in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:28,255 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/a4387231906341ebb3e84b1a8a394945, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/6b1583e2ba4e4ead832691e829d0193d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/8f0bfee0383345028d3bfdf3ac617ed5] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=41.6 K 2024-11-15T23:15:28,255 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/35f1d60c4f3342c8bbd1302ffab88afd, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/193fcc3587964c958e28f10f7624a7b7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/d84ee5ef97ea42808d02a6cbc194e674] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=36.8 K 2024-11-15T23:15:28,255 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 35f1d60c4f3342c8bbd1302ffab88afd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731712526227 2024-11-15T23:15:28,255 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4387231906341ebb3e84b1a8a394945, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731712526227 2024-11-15T23:15:28,256 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b1583e2ba4e4ead832691e829d0193d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731712526883 2024-11-15T23:15:28,256 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 193fcc3587964c958e28f10f7624a7b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731712526886 2024-11-15T23:15:28,257 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f0bfee0383345028d3bfdf3ac617ed5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731712527180 2024-11-15T23:15:28,257 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting d84ee5ef97ea42808d02a6cbc194e674, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731712527180 2024-11-15T23:15:28,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:15:28,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:28,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:28,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:28,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:28,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:28,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/828ab71ad0be430d991fcac72125f5ba is 50, key is test_row_0/A:col10/1731712527329/Put/seqid=0 2024-11-15T23:15:28,287 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#A#compaction#80 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:28,287 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/647327fbf36c4c049c5ea41eecafb8f1 is 50, key is test_row_0/A:col10/1731712527283/Put/seqid=0 2024-11-15T23:15:28,291 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#B#compaction#79 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:28,292 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/8368ec37e7174a59b2c5f6e9b2189f95 is 50, key is test_row_0/B:col10/1731712527283/Put/seqid=0 2024-11-15T23:15:28,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741915_1091 (size=14741) 2024-11-15T23:15:28,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/828ab71ad0be430d991fcac72125f5ba 2024-11-15T23:15:28,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741917_1093 (size=13187) 2024-11-15T23:15:28,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741916_1092 (size=13187) 2024-11-15T23:15:28,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/0e7373b511414a35a98f6215dceaf270 is 50, key is test_row_0/B:col10/1731712527329/Put/seqid=0 2024-11-15T23:15:28,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741918_1094 (size=12301) 2024-11-15T23:15:28,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/0e7373b511414a35a98f6215dceaf270 2024-11-15T23:15:28,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/df20849d661d492dbac2965450bbcbee is 50, key is test_row_0/C:col10/1731712527329/Put/seqid=0 2024-11-15T23:15:28,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741919_1095 (size=12301) 2024-11-15T23:15:28,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712588336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:28,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:28,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:15:28,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:28,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:28,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:28,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:28,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:28,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712588445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:28,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:28,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:28,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:28,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712588475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:28,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712588486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:28,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712588486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:28,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:15:28,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:28,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:28,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:28,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:28,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:28,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712588655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:28,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:28,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:28,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:28,709 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/8368ec37e7174a59b2c5f6e9b2189f95 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/8368ec37e7174a59b2c5f6e9b2189f95 2024-11-15T23:15:28,721 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/B of b9ac2b510c393ceae30043c6a9eaa588 into 8368ec37e7174a59b2c5f6e9b2189f95(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:28,721 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:28,722 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/B, priority=13, startTime=1731712528252; duration=0sec 2024-11-15T23:15:28,722 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:28,722 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:B 2024-11-15T23:15:28,722 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:28,730 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/647327fbf36c4c049c5ea41eecafb8f1 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/647327fbf36c4c049c5ea41eecafb8f1 2024-11-15T23:15:28,731 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:28,731 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/C is initiating minor compaction (all files) 2024-11-15T23:15:28,731 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/C in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:28,731 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/53f1656164e64494be4ffb50e9ac8a7e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/a16b1540936f49209823d28107269962, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/e0d6f3eb0449482aa72cf24b95378595] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=36.8 K 2024-11-15T23:15:28,732 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 53f1656164e64494be4ffb50e9ac8a7e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731712526227 2024-11-15T23:15:28,732 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting a16b1540936f49209823d28107269962, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731712526886 2024-11-15T23:15:28,733 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting e0d6f3eb0449482aa72cf24b95378595, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731712527180 2024-11-15T23:15:28,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/df20849d661d492dbac2965450bbcbee 2024-11-15T23:15:28,739 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/A of b9ac2b510c393ceae30043c6a9eaa588 into 647327fbf36c4c049c5ea41eecafb8f1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:28,739 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:28,739 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/A, priority=13, startTime=1731712528252; duration=0sec 2024-11-15T23:15:28,739 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:28,739 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:A 2024-11-15T23:15:28,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:15:28,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:28,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:28,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:28,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:28,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:28,748 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#C#compaction#83 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:28,749 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/fa9b2b7f64ed461cba8886a1282dab13 is 50, key is test_row_0/C:col10/1731712527283/Put/seqid=0 2024-11-15T23:15:28,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/828ab71ad0be430d991fcac72125f5ba as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/828ab71ad0be430d991fcac72125f5ba 2024-11-15T23:15:28,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/828ab71ad0be430d991fcac72125f5ba, entries=200, sequenceid=364, filesize=14.4 K 2024-11-15T23:15:28,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/0e7373b511414a35a98f6215dceaf270 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/0e7373b511414a35a98f6215dceaf270 2024-11-15T23:15:28,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741920_1096 (size=13187) 2024-11-15T23:15:28,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/0e7373b511414a35a98f6215dceaf270, entries=150, sequenceid=364, filesize=12.0 K 2024-11-15T23:15:28,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/df20849d661d492dbac2965450bbcbee as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/df20849d661d492dbac2965450bbcbee 2024-11-15T23:15:28,796 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/fa9b2b7f64ed461cba8886a1282dab13 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/fa9b2b7f64ed461cba8886a1282dab13 2024-11-15T23:15:28,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/df20849d661d492dbac2965450bbcbee, entries=150, sequenceid=364, filesize=12.0 K 2024-11-15T23:15:28,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b9ac2b510c393ceae30043c6a9eaa588 in 567ms, sequenceid=364, compaction requested=false 2024-11-15T23:15:28,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:28,825 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/C of b9ac2b510c393ceae30043c6a9eaa588 into fa9b2b7f64ed461cba8886a1282dab13(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:28,825 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:28,825 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/C, priority=13, startTime=1731712528252; duration=0sec 2024-11-15T23:15:28,825 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:28,825 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:C 2024-11-15T23:15:28,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:15:28,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:28,900 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T23:15:28,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:28,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:28,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:28,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:28,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:28,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:28,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/c78a4611a4a04a9db91b81af3edb24b0 is 50, key is test_row_0/A:col10/1731712528328/Put/seqid=0 2024-11-15T23:15:28,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741921_1097 (size=12301) 2024-11-15T23:15:28,924 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/c78a4611a4a04a9db91b81af3edb24b0 2024-11-15T23:15:28,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/d757583c828e4349b0e0d80ab9ae4a9f is 50, key is test_row_0/B:col10/1731712528328/Put/seqid=0 2024-11-15T23:15:28,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. as already flushing 2024-11-15T23:15:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:28,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741922_1098 (size=12301) 2024-11-15T23:15:29,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T23:15:29,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712589045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:29,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712589166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:29,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:29,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712589255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:29,261 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_0' in b9ac2b510c393ceae30043c6a9eaa588 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:15:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:29,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712589375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:29,381 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/d757583c828e4349b0e0d80ab9ae4a9f 2024-11-15T23:15:29,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/e6152bb5ee1c4bdc950d53734b91cdc0 is 50, key is test_row_0/C:col10/1731712528328/Put/seqid=0 2024-11-15T23:15:29,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741923_1099 (size=12301) 2024-11-15T23:15:29,439 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/e6152bb5ee1c4bdc950d53734b91cdc0 2024-11-15T23:15:29,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/c78a4611a4a04a9db91b81af3edb24b0 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/c78a4611a4a04a9db91b81af3edb24b0 2024-11-15T23:15:29,466 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/c78a4611a4a04a9db91b81af3edb24b0, entries=150, sequenceid=387, filesize=12.0 K 2024-11-15T23:15:29,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/d757583c828e4349b0e0d80ab9ae4a9f as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/d757583c828e4349b0e0d80ab9ae4a9f 2024-11-15T23:15:29,480 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/d757583c828e4349b0e0d80ab9ae4a9f, entries=150, sequenceid=387, filesize=12.0 K 2024-11-15T23:15:29,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/e6152bb5ee1c4bdc950d53734b91cdc0 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/e6152bb5ee1c4bdc950d53734b91cdc0 2024-11-15T23:15:29,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712589487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:29,489 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in b9ac2b510c393ceae30043c6a9eaa588 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:15:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:29,495 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/e6152bb5ee1c4bdc950d53734b91cdc0, entries=150, sequenceid=387, filesize=12.0 K 2024-11-15T23:15:29,497 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b9ac2b510c393ceae30043c6a9eaa588 in 596ms, sequenceid=387, compaction requested=true 2024-11-15T23:15:29,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:29,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:29,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-15T23:15:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-15T23:15:29,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-15T23:15:29,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6220 sec 2024-11-15T23:15:29,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T23:15:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:29,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:29,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:29,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:29,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:29,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:29,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:29,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.6370 sec 2024-11-15T23:15:29,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/28a1de22cedb42f6b7650d23b3e6d0d8 is 50, key is test_row_0/A:col10/1731712529498/Put/seqid=0 2024-11-15T23:15:29,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741924_1100 (size=14741) 2024-11-15T23:15:29,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712589603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:29,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:29,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712589604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:29,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:29,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:29,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712589686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:29,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712589716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:29,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:29,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712589716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:29,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:29,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712589925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:29,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712589926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:29,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/28a1de22cedb42f6b7650d23b3e6d0d8 2024-11-15T23:15:29,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/efe64721fed241d2b15c4e69064987a8 is 50, key is test_row_0/B:col10/1731712529498/Put/seqid=0 2024-11-15T23:15:30,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741925_1101 (size=12301) 2024-11-15T23:15:30,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/efe64721fed241d2b15c4e69064987a8 2024-11-15T23:15:30,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/c58540aaa3ec43caab33d697b5ae9531 is 50, key is test_row_0/C:col10/1731712529498/Put/seqid=0 2024-11-15T23:15:30,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741926_1102 (size=12301) 2024-11-15T23:15:30,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712590207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:30,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:30,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:30,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:30,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712590235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:30,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:30,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:30,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:30,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712590236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:30,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:30,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:30,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:30,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/c58540aaa3ec43caab33d697b5ae9531 2024-11-15T23:15:30,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/28a1de22cedb42f6b7650d23b3e6d0d8 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/28a1de22cedb42f6b7650d23b3e6d0d8 2024-11-15T23:15:30,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/28a1de22cedb42f6b7650d23b3e6d0d8, entries=200, sequenceid=401, filesize=14.4 K 2024-11-15T23:15:30,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/efe64721fed241d2b15c4e69064987a8 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/efe64721fed241d2b15c4e69064987a8 2024-11-15T23:15:30,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/efe64721fed241d2b15c4e69064987a8, entries=150, sequenceid=401, filesize=12.0 K 2024-11-15T23:15:30,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/c58540aaa3ec43caab33d697b5ae9531 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/c58540aaa3ec43caab33d697b5ae9531 2024-11-15T23:15:30,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/c58540aaa3ec43caab33d697b5ae9531, entries=150, sequenceid=401, filesize=12.0 K 2024-11-15T23:15:30,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for b9ac2b510c393ceae30043c6a9eaa588 in 979ms, sequenceid=401, compaction requested=true 2024-11-15T23:15:30,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:30,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:15:30,481 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:15:30,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:30,482 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:15:30,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:15:30,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:30,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:15:30,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:30,483 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:15:30,483 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/A is initiating minor compaction (all files) 2024-11-15T23:15:30,483 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/A in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:30,484 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/647327fbf36c4c049c5ea41eecafb8f1, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/828ab71ad0be430d991fcac72125f5ba, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/c78a4611a4a04a9db91b81af3edb24b0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/28a1de22cedb42f6b7650d23b3e6d0d8] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=53.7 K 2024-11-15T23:15:30,484 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:15:30,484 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/B is initiating minor compaction (all files) 2024-11-15T23:15:30,484 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/B in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:30,484 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/8368ec37e7174a59b2c5f6e9b2189f95, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/0e7373b511414a35a98f6215dceaf270, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/d757583c828e4349b0e0d80ab9ae4a9f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/efe64721fed241d2b15c4e69064987a8] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=48.9 K 2024-11-15T23:15:30,484 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 647327fbf36c4c049c5ea41eecafb8f1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731712527180 2024-11-15T23:15:30,485 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 8368ec37e7174a59b2c5f6e9b2189f95, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731712527180 2024-11-15T23:15:30,485 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 828ab71ad0be430d991fcac72125f5ba, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731712527309 2024-11-15T23:15:30,485 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e7373b511414a35a98f6215dceaf270, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731712527322 2024-11-15T23:15:30,485 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting c78a4611a4a04a9db91b81af3edb24b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731712528321 2024-11-15T23:15:30,485 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting d757583c828e4349b0e0d80ab9ae4a9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731712528321 2024-11-15T23:15:30,486 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28a1de22cedb42f6b7650d23b3e6d0d8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731712529020 2024-11-15T23:15:30,486 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting efe64721fed241d2b15c4e69064987a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731712529020 2024-11-15T23:15:30,496 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#A#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:30,497 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#B#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:30,497 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/11368f59a5414abeb6a7dd53c8d5581a is 50, key is test_row_0/A:col10/1731712529498/Put/seqid=0 2024-11-15T23:15:30,497 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/62414abeae384f06991db7a66bf08ee9 is 50, key is test_row_0/B:col10/1731712529498/Put/seqid=0 2024-11-15T23:15:30,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741927_1103 (size=13323) 2024-11-15T23:15:30,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741928_1104 (size=13323) 2024-11-15T23:15:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:30,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T23:15:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:30,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/a23fe582b53f4227ae8346e1f8da950b is 50, key is test_row_0/A:col10/1731712530746/Put/seqid=0 2024-11-15T23:15:30,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741929_1105 (size=14741) 2024-11-15T23:15:30,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/a23fe582b53f4227ae8346e1f8da950b 2024-11-15T23:15:30,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/dad6a37f0bff487d876e82bc7f5baf03 is 50, key is test_row_0/B:col10/1731712530746/Put/seqid=0 2024-11-15T23:15:30,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712590775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:30,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712590776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:30,780 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-15T23:15:30,780 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-15T23:15:30,780 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:15:30,780 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:15:30,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741930_1106 (size=12301) 2024-11-15T23:15:30,785 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-15T23:15:30,785 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:15:30,786 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:30,786 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:30,786 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:30,787 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:30,787 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:30,787 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:30,787 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:15:30,787 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:15:30,787 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:15:30,793 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-15T23:15:30,793 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:15:30,793 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:30,793 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:30,793 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:15:30,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:30,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712590886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712590886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:30,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:30,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:30,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:30,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:30,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:30,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:30,925 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/62414abeae384f06991db7a66bf08ee9 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/62414abeae384f06991db7a66bf08ee9 2024-11-15T23:15:30,927 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/11368f59a5414abeb6a7dd53c8d5581a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/11368f59a5414abeb6a7dd53c8d5581a 2024-11-15T23:15:30,932 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/B of b9ac2b510c393ceae30043c6a9eaa588 into 62414abeae384f06991db7a66bf08ee9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:30,932 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:30,932 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/B, priority=12, startTime=1731712530481; duration=0sec 2024-11-15T23:15:30,932 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:30,932 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:B 2024-11-15T23:15:30,932 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:15:30,933 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/A of b9ac2b510c393ceae30043c6a9eaa588 into 11368f59a5414abeb6a7dd53c8d5581a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:30,933 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:30,933 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/A, priority=12, startTime=1731712530481; duration=0sec 2024-11-15T23:15:30,933 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:30,933 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:A 2024-11-15T23:15:30,934 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:15:30,934 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/C is initiating minor compaction (all files) 2024-11-15T23:15:30,934 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/C in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:30,934 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/fa9b2b7f64ed461cba8886a1282dab13, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/df20849d661d492dbac2965450bbcbee, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/e6152bb5ee1c4bdc950d53734b91cdc0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/c58540aaa3ec43caab33d697b5ae9531] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=48.9 K 2024-11-15T23:15:30,935 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting fa9b2b7f64ed461cba8886a1282dab13, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731712527180 2024-11-15T23:15:30,935 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting df20849d661d492dbac2965450bbcbee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731712527322 2024-11-15T23:15:30,935 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting e6152bb5ee1c4bdc950d53734b91cdc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731712528321 2024-11-15T23:15:30,936 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting c58540aaa3ec43caab33d697b5ae9531, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731712529020 2024-11-15T23:15:30,943 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#C#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:30,943 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/f50ae6fea47b40abbb215a8c8ae59dad is 50, key is test_row_0/C:col10/1731712529498/Put/seqid=0 2024-11-15T23:15:30,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741931_1107 (size=13323) 2024-11-15T23:15:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T23:15:31,037 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:31,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:31,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712591096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712591097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:31,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/dad6a37f0bff487d876e82bc7f5baf03 2024-11-15T23:15:31,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/3e1dc691fdaa4286a13ab2d7a70e36d0 is 50, key is test_row_0/C:col10/1731712530746/Put/seqid=0 2024-11-15T23:15:31,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741932_1108 (size=12301) 2024-11-15T23:15:31,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43560 deadline: 1731712591225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:31,226 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_0' in b9ac2b510c393ceae30043c6a9eaa588 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:15:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:31,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1731712591276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:31,277 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_0' in b9ac2b510c393ceae30043c6a9eaa588 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:15:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:31,361 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/f50ae6fea47b40abbb215a8c8ae59dad as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/f50ae6fea47b40abbb215a8c8ae59dad 2024-11-15T23:15:31,367 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/C of b9ac2b510c393ceae30043c6a9eaa588 into f50ae6fea47b40abbb215a8c8ae59dad(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:31,367 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:31,367 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/C, priority=12, startTime=1731712530482; duration=0sec 2024-11-15T23:15:31,367 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:31,367 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:C 2024-11-15T23:15:31,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:31,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43552 deadline: 1731712591405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43612 deadline: 1731712591406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:31,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:31,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:31,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43580 deadline: 1731712591505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:31,506 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in b9ac2b510c393ceae30043c6a9eaa588 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:15:31,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:31,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:31,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:15:31,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/3e1dc691fdaa4286a13ab2d7a70e36d0 2024-11-15T23:15:31,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/a23fe582b53f4227ae8346e1f8da950b as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/a23fe582b53f4227ae8346e1f8da950b 2024-11-15T23:15:31,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/a23fe582b53f4227ae8346e1f8da950b, entries=200, sequenceid=421, filesize=14.4 K 2024-11-15T23:15:31,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/dad6a37f0bff487d876e82bc7f5baf03 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/dad6a37f0bff487d876e82bc7f5baf03 2024-11-15T23:15:31,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/dad6a37f0bff487d876e82bc7f5baf03, entries=150, sequenceid=421, filesize=12.0 K 2024-11-15T23:15:31,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/3e1dc691fdaa4286a13ab2d7a70e36d0 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/3e1dc691fdaa4286a13ab2d7a70e36d0 2024-11-15T23:15:31,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/3e1dc691fdaa4286a13ab2d7a70e36d0, entries=150, sequenceid=421, filesize=12.0 K 2024-11-15T23:15:31,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b9ac2b510c393ceae30043c6a9eaa588 in 882ms, sequenceid=421, compaction requested=false 2024-11-15T23:15:31,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:31,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:31,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T23:15:31,921 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-15T23:15:31,921 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:15:31,921 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:31,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:31,922 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-15T23:15:31,922 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:31,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:31,922 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:15:31,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:31,922 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:15:31,922 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:31,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:31,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:31,922 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:31,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:31,922 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:15:31,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/5e62a9d861284ad787b4b454f18add7e is 50, key is test_row_0/A:col10/1731712530758/Put/seqid=0 2024-11-15T23:15:31,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741933_1109 (size=12301) 2024-11-15T23:15:31,967 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T23:15:32,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/5e62a9d861284ad787b4b454f18add7e 2024-11-15T23:15:32,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/9823ab489ef44988a2704ceb909524d1 is 50, key is test_row_0/B:col10/1731712530758/Put/seqid=0 2024-11-15T23:15:32,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741934_1110 (size=12301) 2024-11-15T23:15:32,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/9823ab489ef44988a2704ceb909524d1 2024-11-15T23:15:32,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/9d6698b8d9104bdabee3c5ec44ee0c67 is 50, key is test_row_0/C:col10/1731712530758/Put/seqid=0 2024-11-15T23:15:32,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741935_1111 (size=12301) 2024-11-15T23:15:33,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/9d6698b8d9104bdabee3c5ec44ee0c67 2024-11-15T23:15:33,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/5e62a9d861284ad787b4b454f18add7e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/5e62a9d861284ad787b4b454f18add7e 2024-11-15T23:15:33,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/5e62a9d861284ad787b4b454f18add7e, entries=150, sequenceid=438, filesize=12.0 K 2024-11-15T23:15:33,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/9823ab489ef44988a2704ceb909524d1 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/9823ab489ef44988a2704ceb909524d1 2024-11-15T23:15:33,248 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-15T23:15:33,248 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:15:33,248 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:33,248 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:33,248 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:15:33,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/9823ab489ef44988a2704ceb909524d1, entries=150, sequenceid=438, filesize=12.0 K 2024-11-15T23:15:33,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/9d6698b8d9104bdabee3c5ec44ee0c67 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/9d6698b8d9104bdabee3c5ec44ee0c67 2024-11-15T23:15:33,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/9d6698b8d9104bdabee3c5ec44ee0c67, entries=150, sequenceid=438, filesize=12.0 K 2024-11-15T23:15:33,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=6.71 KB/6870 for b9ac2b510c393ceae30043c6a9eaa588 in 1334ms, sequenceid=438, compaction requested=true 2024-11-15T23:15:33,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:33,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:15:33,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:33,256 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:33,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:15:33,256 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:33,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:33,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac2b510c393ceae30043c6a9eaa588:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:15:33,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:33,257 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:33,257 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:33,257 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/B is initiating minor compaction (all files) 2024-11-15T23:15:33,257 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/A is initiating minor compaction (all files) 2024-11-15T23:15:33,257 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/B in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:33,257 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/A in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:33,257 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/11368f59a5414abeb6a7dd53c8d5581a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/a23fe582b53f4227ae8346e1f8da950b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/5e62a9d861284ad787b4b454f18add7e] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=39.4 K 2024-11-15T23:15:33,257 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/62414abeae384f06991db7a66bf08ee9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/dad6a37f0bff487d876e82bc7f5baf03, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/9823ab489ef44988a2704ceb909524d1] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=37.0 K 2024-11-15T23:15:33,258 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11368f59a5414abeb6a7dd53c8d5581a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731712529020 2024-11-15T23:15:33,258 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 62414abeae384f06991db7a66bf08ee9, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731712529020 2024-11-15T23:15:33,258 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting a23fe582b53f4227ae8346e1f8da950b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731712529593 2024-11-15T23:15:33,258 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting dad6a37f0bff487d876e82bc7f5baf03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731712529593 2024-11-15T23:15:33,258 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 9823ab489ef44988a2704ceb909524d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731712530758 2024-11-15T23:15:33,258 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e62a9d861284ad787b4b454f18add7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731712530758 2024-11-15T23:15:33,267 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#A#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:33,267 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/a93b50e1f4854cb8b7d12715e64db311 is 50, key is test_row_0/A:col10/1731712530758/Put/seqid=0 2024-11-15T23:15:33,269 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#B#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:33,270 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/42a8d43ae37d4ce49a9ef05b7e20ca71 is 50, key is test_row_0/B:col10/1731712530758/Put/seqid=0 2024-11-15T23:15:33,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741936_1112 (size=13425) 2024-11-15T23:15:33,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741937_1113 (size=13425) 2024-11-15T23:15:33,686 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/42a8d43ae37d4ce49a9ef05b7e20ca71 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/42a8d43ae37d4ce49a9ef05b7e20ca71 2024-11-15T23:15:33,686 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/a93b50e1f4854cb8b7d12715e64db311 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/a93b50e1f4854cb8b7d12715e64db311 2024-11-15T23:15:33,692 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/B of b9ac2b510c393ceae30043c6a9eaa588 into 42a8d43ae37d4ce49a9ef05b7e20ca71(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:33,692 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/A of b9ac2b510c393ceae30043c6a9eaa588 into a93b50e1f4854cb8b7d12715e64db311(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:33,692 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:33,692 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:33,692 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/B, priority=13, startTime=1731712533256; duration=0sec 2024-11-15T23:15:33,692 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/A, priority=13, startTime=1731712533256; duration=0sec 2024-11-15T23:15:33,692 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:33,692 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:B 2024-11-15T23:15:33,692 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:33,692 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:A 2024-11-15T23:15:33,692 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:33,693 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:33,693 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b9ac2b510c393ceae30043c6a9eaa588/C is initiating minor compaction (all files) 2024-11-15T23:15:33,693 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac2b510c393ceae30043c6a9eaa588/C in TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:33,693 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/f50ae6fea47b40abbb215a8c8ae59dad, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/3e1dc691fdaa4286a13ab2d7a70e36d0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/9d6698b8d9104bdabee3c5ec44ee0c67] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp, totalSize=37.0 K 2024-11-15T23:15:33,694 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting f50ae6fea47b40abbb215a8c8ae59dad, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731712529020 2024-11-15T23:15:33,694 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e1dc691fdaa4286a13ab2d7a70e36d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731712529593 2024-11-15T23:15:33,695 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d6698b8d9104bdabee3c5ec44ee0c67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731712530758 2024-11-15T23:15:33,704 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac2b510c393ceae30043c6a9eaa588#C#compaction#101 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:33,704 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/abc8e714a19047e3b5b98b7eeaa3d68c is 50, key is test_row_0/C:col10/1731712530758/Put/seqid=0 2024-11-15T23:15:33,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741938_1114 (size=13425) 2024-11-15T23:15:34,121 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/abc8e714a19047e3b5b98b7eeaa3d68c as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/abc8e714a19047e3b5b98b7eeaa3d68c 2024-11-15T23:15:34,131 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac2b510c393ceae30043c6a9eaa588/C of b9ac2b510c393ceae30043c6a9eaa588 into abc8e714a19047e3b5b98b7eeaa3d68c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:34,131 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac2b510c393ceae30043c6a9eaa588: 2024-11-15T23:15:34,132 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., storeName=b9ac2b510c393ceae30043c6a9eaa588/C, priority=13, startTime=1731712533256; duration=0sec 2024-11-15T23:15:34,132 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:34,132 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac2b510c393ceae30043c6a9eaa588:C 2024-11-15T23:15:35,301 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-15T23:15:35,302 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:15:35,302 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:35,302 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:35,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:15:35,529 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-15T23:15:35,529 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:15:35,530 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:35,530 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:35,530 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:15:35,531 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T23:15:35,531 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-15T23:15:35,531 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-15T23:15:35,531 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-15T23:15:35,531 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-15T23:15:35,531 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-15T23:15:35,532 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T23:15:35,532 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2700 2024-11-15T23:15:35,532 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2663 2024-11-15T23:15:35,532 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T23:15:35,532 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1219 2024-11-15T23:15:35,532 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3652 rows 2024-11-15T23:15:35,532 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1219 2024-11-15T23:15:35,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3652 rows 2024-11-15T23:15:35,533 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T23:15:35,533 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T23:15:35,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:35,534 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:35,534 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:15:35,538 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T23:15:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T23:15:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T23:15:35,545 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712535544"}]},"ts":"1731712535544"} 2024-11-15T23:15:35,547 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T23:15:35,547 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T23:15:35,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T23:15:35,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9ac2b510c393ceae30043c6a9eaa588, UNASSIGN}] 2024-11-15T23:15:35,557 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9ac2b510c393ceae30043c6a9eaa588, UNASSIGN 2024-11-15T23:15:35,558 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=b9ac2b510c393ceae30043c6a9eaa588, regionState=CLOSING, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:35,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9ac2b510c393ceae30043c6a9eaa588, UNASSIGN because future has completed 2024-11-15T23:15:35,561 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T23:15:35,561 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763}] 2024-11-15T23:15:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T23:15:35,720 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:35,721 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T23:15:35,722 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing b9ac2b510c393ceae30043c6a9eaa588, disabling compactions & flushes 2024-11-15T23:15:35,722 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:35,722 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:35,722 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. after waiting 0 ms 2024-11-15T23:15:35,722 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:35,723 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing b9ac2b510c393ceae30043c6a9eaa588 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-15T23:15:35,723 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=A 2024-11-15T23:15:35,723 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:35,723 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=B 2024-11-15T23:15:35,723 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:35,724 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b9ac2b510c393ceae30043c6a9eaa588, store=C 2024-11-15T23:15:35,724 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:35,729 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/5113b8131c8c438aa2ddc907aab8e055 is 50, key is test_row_0/A:col10/1731712535298/Put/seqid=0 2024-11-15T23:15:35,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741939_1115 (size=9857) 2024-11-15T23:15:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T23:15:36,136 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/5113b8131c8c438aa2ddc907aab8e055 2024-11-15T23:15:36,145 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/e00a213137c14bb9a554899aacc2aba3 is 50, key is test_row_0/B:col10/1731712535298/Put/seqid=0 2024-11-15T23:15:36,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741940_1116 (size=9857) 2024-11-15T23:15:36,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T23:15:36,553 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/e00a213137c14bb9a554899aacc2aba3 2024-11-15T23:15:36,563 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/2920b75032ab4a6fb8951f8328bb5509 is 50, key is test_row_0/C:col10/1731712535298/Put/seqid=0 2024-11-15T23:15:36,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741941_1117 (size=9857) 2024-11-15T23:15:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T23:15:36,968 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/2920b75032ab4a6fb8951f8328bb5509 2024-11-15T23:15:36,974 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/A/5113b8131c8c438aa2ddc907aab8e055 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/5113b8131c8c438aa2ddc907aab8e055 2024-11-15T23:15:36,978 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/5113b8131c8c438aa2ddc907aab8e055, entries=100, sequenceid=447, filesize=9.6 K 2024-11-15T23:15:36,979 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/B/e00a213137c14bb9a554899aacc2aba3 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/e00a213137c14bb9a554899aacc2aba3 2024-11-15T23:15:36,984 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/e00a213137c14bb9a554899aacc2aba3, entries=100, sequenceid=447, filesize=9.6 K 2024-11-15T23:15:36,985 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/.tmp/C/2920b75032ab4a6fb8951f8328bb5509 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/2920b75032ab4a6fb8951f8328bb5509 2024-11-15T23:15:36,990 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/2920b75032ab4a6fb8951f8328bb5509, entries=100, sequenceid=447, filesize=9.6 K 2024-11-15T23:15:36,991 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for b9ac2b510c393ceae30043c6a9eaa588 in 1269ms, sequenceid=447, compaction requested=false 2024-11-15T23:15:36,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/86135e59455746a0b1669d265454d97a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/262c1611a77e47199ae8dac7e7e54790, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/3544a0a082884526a3c2adfcea6cc46d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/1337d2b9105c4081ab9b30df2e6d3270, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/9ad5010c46e24b34afe4d0f78578e70e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/57e4f44c4a9443a0b8781a7eed2dfc2f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/5712a6cf5a1f43d1ab091fa936109ef5, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/6746ccf2504a4fceaacdb04819732e80, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/0a92f1211fed440b8584ee2b472569af, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/e0e91254a90a46e6ae24a0b0230dec55, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/8959b43e8e6f42fda6a25646d570cdc5, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/2f8836115e644f93a8e9d264f324ea0b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/338024f06c4342dbac1a68c61580fa49, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/1169f13a7ff74f048431a8d6430a058e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/825b01e8e58d4475866d75d71ce65f1e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/f6f504a112814ba086a3498c1a4f8593, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/4f345c2ae5984108a3cd21382bf6889e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/9df6b5d8da08460c824c70dbca699780, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/8d11331ab19643abac0068be95ed4c2e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/71c2aab306a740f48c06feb9014c83a1, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/c6c11f86e7f046ca82fb6bad49f2b35d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/592fdcdad13744909406446d92875d2d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/a4387231906341ebb3e84b1a8a394945, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/682e2bd73a014f8d99e27d2279433e00, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/6b1583e2ba4e4ead832691e829d0193d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/8f0bfee0383345028d3bfdf3ac617ed5, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/647327fbf36c4c049c5ea41eecafb8f1, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/828ab71ad0be430d991fcac72125f5ba, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/c78a4611a4a04a9db91b81af3edb24b0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/28a1de22cedb42f6b7650d23b3e6d0d8, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/11368f59a5414abeb6a7dd53c8d5581a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/a23fe582b53f4227ae8346e1f8da950b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/5e62a9d861284ad787b4b454f18add7e] to archive 2024-11-15T23:15:36,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:15:37,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/86135e59455746a0b1669d265454d97a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/86135e59455746a0b1669d265454d97a 2024-11-15T23:15:37,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/262c1611a77e47199ae8dac7e7e54790 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/262c1611a77e47199ae8dac7e7e54790 2024-11-15T23:15:37,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/3544a0a082884526a3c2adfcea6cc46d to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/3544a0a082884526a3c2adfcea6cc46d 2024-11-15T23:15:37,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/1337d2b9105c4081ab9b30df2e6d3270 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/1337d2b9105c4081ab9b30df2e6d3270 2024-11-15T23:15:37,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/9ad5010c46e24b34afe4d0f78578e70e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/9ad5010c46e24b34afe4d0f78578e70e 2024-11-15T23:15:37,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/57e4f44c4a9443a0b8781a7eed2dfc2f to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/57e4f44c4a9443a0b8781a7eed2dfc2f 2024-11-15T23:15:37,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/5712a6cf5a1f43d1ab091fa936109ef5 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/5712a6cf5a1f43d1ab091fa936109ef5 2024-11-15T23:15:37,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/6746ccf2504a4fceaacdb04819732e80 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/6746ccf2504a4fceaacdb04819732e80 2024-11-15T23:15:37,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/0a92f1211fed440b8584ee2b472569af to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/0a92f1211fed440b8584ee2b472569af 2024-11-15T23:15:37,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/e0e91254a90a46e6ae24a0b0230dec55 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/e0e91254a90a46e6ae24a0b0230dec55 2024-11-15T23:15:37,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/8959b43e8e6f42fda6a25646d570cdc5 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/8959b43e8e6f42fda6a25646d570cdc5 2024-11-15T23:15:37,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/2f8836115e644f93a8e9d264f324ea0b to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/2f8836115e644f93a8e9d264f324ea0b 2024-11-15T23:15:37,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/338024f06c4342dbac1a68c61580fa49 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/338024f06c4342dbac1a68c61580fa49 2024-11-15T23:15:37,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/1169f13a7ff74f048431a8d6430a058e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/1169f13a7ff74f048431a8d6430a058e 2024-11-15T23:15:37,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/825b01e8e58d4475866d75d71ce65f1e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/825b01e8e58d4475866d75d71ce65f1e 2024-11-15T23:15:37,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/f6f504a112814ba086a3498c1a4f8593 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/f6f504a112814ba086a3498c1a4f8593 2024-11-15T23:15:37,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/4f345c2ae5984108a3cd21382bf6889e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/4f345c2ae5984108a3cd21382bf6889e 2024-11-15T23:15:37,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/9df6b5d8da08460c824c70dbca699780 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/9df6b5d8da08460c824c70dbca699780 2024-11-15T23:15:37,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/8d11331ab19643abac0068be95ed4c2e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/8d11331ab19643abac0068be95ed4c2e 2024-11-15T23:15:37,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/71c2aab306a740f48c06feb9014c83a1 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/71c2aab306a740f48c06feb9014c83a1 2024-11-15T23:15:37,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/c6c11f86e7f046ca82fb6bad49f2b35d to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/c6c11f86e7f046ca82fb6bad49f2b35d 2024-11-15T23:15:37,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/592fdcdad13744909406446d92875d2d to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/592fdcdad13744909406446d92875d2d 2024-11-15T23:15:37,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/a4387231906341ebb3e84b1a8a394945 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/a4387231906341ebb3e84b1a8a394945 2024-11-15T23:15:37,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/682e2bd73a014f8d99e27d2279433e00 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/682e2bd73a014f8d99e27d2279433e00 2024-11-15T23:15:37,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/6b1583e2ba4e4ead832691e829d0193d to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/6b1583e2ba4e4ead832691e829d0193d 2024-11-15T23:15:37,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/8f0bfee0383345028d3bfdf3ac617ed5 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/8f0bfee0383345028d3bfdf3ac617ed5 2024-11-15T23:15:37,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/647327fbf36c4c049c5ea41eecafb8f1 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/647327fbf36c4c049c5ea41eecafb8f1 2024-11-15T23:15:37,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/828ab71ad0be430d991fcac72125f5ba to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/828ab71ad0be430d991fcac72125f5ba 2024-11-15T23:15:37,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/c78a4611a4a04a9db91b81af3edb24b0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/c78a4611a4a04a9db91b81af3edb24b0 2024-11-15T23:15:37,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/28a1de22cedb42f6b7650d23b3e6d0d8 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/28a1de22cedb42f6b7650d23b3e6d0d8 2024-11-15T23:15:37,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/11368f59a5414abeb6a7dd53c8d5581a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/11368f59a5414abeb6a7dd53c8d5581a 2024-11-15T23:15:37,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/a23fe582b53f4227ae8346e1f8da950b to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/a23fe582b53f4227ae8346e1f8da950b 2024-11-15T23:15:37,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/5e62a9d861284ad787b4b454f18add7e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/5e62a9d861284ad787b4b454f18add7e 2024-11-15T23:15:37,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/b300bc1780d348a0836844d613c68d89, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/6478d59005f548679e4f81c09bfbe74e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/3ec627692bd94b21a1236cf5bac34cb6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/4cc4c1ab4fff48dead1394b89efb1de3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/5f42f80aab144ccca99cdbdff10632bf, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/ec6b6ec26a514ab089e78c0aa1924f5b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/f079904b3c54460d883e3bac11b7d6d5, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/f099ee56e1804087b04114a764029da8, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/5147ecba55b943e0a4be48a9e20a3803, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/993ed588f84d4677ade594156432db64, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/e97e0d04df964965b31685ba7fdf7b0c, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/9354c1512dcf44ef8f77d6bbc4f98b1b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/d62a318ef1bd4c78a94c464ea5510ffe, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/58cefde13f7642d3ab7a6fa8c24498ae, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/00fa81ef58e74b2fb26f45e0094916a9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/3a882c04e8f746a08e60a21df64b6f3d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/e411a4018df144d182681119c67e4ac8, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/3cc89c29471349b99da8a1adab9bec90, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/5d3002d17ad04d1d9a31d169ef335e39, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/8edf663d8e764f5796b3c303c1437135, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/6ec20158d3484247819dcb3b7532e82a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/2610b87b1e7f44e9bda2c89621d64e0b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/35f1d60c4f3342c8bbd1302ffab88afd, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/76be7f812ab14a9997153a481bb6c2a3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/193fcc3587964c958e28f10f7624a7b7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/8368ec37e7174a59b2c5f6e9b2189f95, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/d84ee5ef97ea42808d02a6cbc194e674, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/0e7373b511414a35a98f6215dceaf270, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/d757583c828e4349b0e0d80ab9ae4a9f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/62414abeae384f06991db7a66bf08ee9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/efe64721fed241d2b15c4e69064987a8, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/dad6a37f0bff487d876e82bc7f5baf03, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/9823ab489ef44988a2704ceb909524d1] to archive 2024-11-15T23:15:37,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:15:37,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/b300bc1780d348a0836844d613c68d89 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/b300bc1780d348a0836844d613c68d89 2024-11-15T23:15:37,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/6478d59005f548679e4f81c09bfbe74e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/6478d59005f548679e4f81c09bfbe74e 2024-11-15T23:15:37,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/3ec627692bd94b21a1236cf5bac34cb6 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/3ec627692bd94b21a1236cf5bac34cb6 2024-11-15T23:15:37,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/4cc4c1ab4fff48dead1394b89efb1de3 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/4cc4c1ab4fff48dead1394b89efb1de3 2024-11-15T23:15:37,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/5f42f80aab144ccca99cdbdff10632bf to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/5f42f80aab144ccca99cdbdff10632bf 2024-11-15T23:15:37,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/ec6b6ec26a514ab089e78c0aa1924f5b to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/ec6b6ec26a514ab089e78c0aa1924f5b 2024-11-15T23:15:37,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/f079904b3c54460d883e3bac11b7d6d5 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/f079904b3c54460d883e3bac11b7d6d5 2024-11-15T23:15:37,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/f099ee56e1804087b04114a764029da8 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/f099ee56e1804087b04114a764029da8 2024-11-15T23:15:37,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/5147ecba55b943e0a4be48a9e20a3803 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/5147ecba55b943e0a4be48a9e20a3803 2024-11-15T23:15:37,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/993ed588f84d4677ade594156432db64 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/993ed588f84d4677ade594156432db64 2024-11-15T23:15:37,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/e97e0d04df964965b31685ba7fdf7b0c to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/e97e0d04df964965b31685ba7fdf7b0c 2024-11-15T23:15:37,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/9354c1512dcf44ef8f77d6bbc4f98b1b to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/9354c1512dcf44ef8f77d6bbc4f98b1b 2024-11-15T23:15:37,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/d62a318ef1bd4c78a94c464ea5510ffe to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/d62a318ef1bd4c78a94c464ea5510ffe 2024-11-15T23:15:37,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/58cefde13f7642d3ab7a6fa8c24498ae to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/58cefde13f7642d3ab7a6fa8c24498ae 2024-11-15T23:15:37,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/00fa81ef58e74b2fb26f45e0094916a9 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/00fa81ef58e74b2fb26f45e0094916a9 2024-11-15T23:15:37,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/3a882c04e8f746a08e60a21df64b6f3d to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/3a882c04e8f746a08e60a21df64b6f3d 2024-11-15T23:15:37,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/e411a4018df144d182681119c67e4ac8 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/e411a4018df144d182681119c67e4ac8 2024-11-15T23:15:37,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/3cc89c29471349b99da8a1adab9bec90 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/3cc89c29471349b99da8a1adab9bec90 2024-11-15T23:15:37,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/5d3002d17ad04d1d9a31d169ef335e39 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/5d3002d17ad04d1d9a31d169ef335e39 2024-11-15T23:15:37,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/8edf663d8e764f5796b3c303c1437135 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/8edf663d8e764f5796b3c303c1437135 2024-11-15T23:15:37,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/6ec20158d3484247819dcb3b7532e82a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/6ec20158d3484247819dcb3b7532e82a 2024-11-15T23:15:37,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/2610b87b1e7f44e9bda2c89621d64e0b to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/2610b87b1e7f44e9bda2c89621d64e0b 2024-11-15T23:15:37,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/35f1d60c4f3342c8bbd1302ffab88afd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/35f1d60c4f3342c8bbd1302ffab88afd 2024-11-15T23:15:37,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/76be7f812ab14a9997153a481bb6c2a3 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/76be7f812ab14a9997153a481bb6c2a3 2024-11-15T23:15:37,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/193fcc3587964c958e28f10f7624a7b7 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/193fcc3587964c958e28f10f7624a7b7 2024-11-15T23:15:37,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/8368ec37e7174a59b2c5f6e9b2189f95 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/8368ec37e7174a59b2c5f6e9b2189f95 2024-11-15T23:15:37,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/d84ee5ef97ea42808d02a6cbc194e674 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/d84ee5ef97ea42808d02a6cbc194e674 2024-11-15T23:15:37,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/0e7373b511414a35a98f6215dceaf270 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/0e7373b511414a35a98f6215dceaf270 2024-11-15T23:15:37,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/d757583c828e4349b0e0d80ab9ae4a9f to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/d757583c828e4349b0e0d80ab9ae4a9f 2024-11-15T23:15:37,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/62414abeae384f06991db7a66bf08ee9 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/62414abeae384f06991db7a66bf08ee9 2024-11-15T23:15:37,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/efe64721fed241d2b15c4e69064987a8 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/efe64721fed241d2b15c4e69064987a8 2024-11-15T23:15:37,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/dad6a37f0bff487d876e82bc7f5baf03 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/dad6a37f0bff487d876e82bc7f5baf03 2024-11-15T23:15:37,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/9823ab489ef44988a2704ceb909524d1 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/9823ab489ef44988a2704ceb909524d1 2024-11-15T23:15:37,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/568c9f63dfcb4ad6bc7d1944e246124d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/66edd19b46614a6c82b7a6a1f980710a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/798937d1ad244c71a1da5d64ef4544e3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/c5f44ae2a1774bd2a8b6fd7549573b03, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/dd87e457751346908be1f58f4178aa3c, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/9d161af91aa0478f8955497b546d7724, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/0f36319ca2234e8688a4557634b3fd28, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/dfbeb81f559c439b9500c670bd9f3652, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/0a746b4c350d45b09b7817cfc2be8803, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/2082ec15859840ee84682d6a050f697a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/db3a22c42b01488eaa305e499bf89df9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/a1e376d914e24483a9c07f3677f5e6d0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/6348b33c4f2d4598be2dd615fb424e19, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/789fbc5c8eb74d0dafb6f199a9a2045d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/54ec0ac2fb7140229df85e35cebda48e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/f65dbcf2a1824062ada229aa59b02531, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/3172e9e9117e4568beb4e51c1bea1cc3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/69c9410024954107b29e53e3f8cd9cfa, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/7ce04c434a95480e92b949596c314276, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/fa4eec9e166542119aff1e0c011916bd, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/a5b654005fe64a749c982ed270c5dd0d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/55f8de73703f4e57928c69fd0b811bad, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/53f1656164e64494be4ffb50e9ac8a7e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/5f4bef66d77e440bb2c77069b810a1dc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/a16b1540936f49209823d28107269962, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/fa9b2b7f64ed461cba8886a1282dab13, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/e0d6f3eb0449482aa72cf24b95378595, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/df20849d661d492dbac2965450bbcbee, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/e6152bb5ee1c4bdc950d53734b91cdc0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/f50ae6fea47b40abbb215a8c8ae59dad, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/c58540aaa3ec43caab33d697b5ae9531, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/3e1dc691fdaa4286a13ab2d7a70e36d0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/9d6698b8d9104bdabee3c5ec44ee0c67] to archive 2024-11-15T23:15:37,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:15:37,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/568c9f63dfcb4ad6bc7d1944e246124d to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/568c9f63dfcb4ad6bc7d1944e246124d 2024-11-15T23:15:37,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/66edd19b46614a6c82b7a6a1f980710a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/66edd19b46614a6c82b7a6a1f980710a 2024-11-15T23:15:37,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/798937d1ad244c71a1da5d64ef4544e3 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/798937d1ad244c71a1da5d64ef4544e3 2024-11-15T23:15:37,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/c5f44ae2a1774bd2a8b6fd7549573b03 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/c5f44ae2a1774bd2a8b6fd7549573b03 2024-11-15T23:15:37,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/dd87e457751346908be1f58f4178aa3c to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/dd87e457751346908be1f58f4178aa3c 2024-11-15T23:15:37,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/9d161af91aa0478f8955497b546d7724 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/9d161af91aa0478f8955497b546d7724 2024-11-15T23:15:37,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/0f36319ca2234e8688a4557634b3fd28 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/0f36319ca2234e8688a4557634b3fd28 2024-11-15T23:15:37,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/dfbeb81f559c439b9500c670bd9f3652 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/dfbeb81f559c439b9500c670bd9f3652 2024-11-15T23:15:37,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/0a746b4c350d45b09b7817cfc2be8803 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/0a746b4c350d45b09b7817cfc2be8803 2024-11-15T23:15:37,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/2082ec15859840ee84682d6a050f697a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/2082ec15859840ee84682d6a050f697a 2024-11-15T23:15:37,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/db3a22c42b01488eaa305e499bf89df9 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/db3a22c42b01488eaa305e499bf89df9 2024-11-15T23:15:37,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/a1e376d914e24483a9c07f3677f5e6d0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/a1e376d914e24483a9c07f3677f5e6d0 2024-11-15T23:15:37,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/6348b33c4f2d4598be2dd615fb424e19 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/6348b33c4f2d4598be2dd615fb424e19 2024-11-15T23:15:37,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/789fbc5c8eb74d0dafb6f199a9a2045d to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/789fbc5c8eb74d0dafb6f199a9a2045d 2024-11-15T23:15:37,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/54ec0ac2fb7140229df85e35cebda48e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/54ec0ac2fb7140229df85e35cebda48e 2024-11-15T23:15:37,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/f65dbcf2a1824062ada229aa59b02531 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/f65dbcf2a1824062ada229aa59b02531 2024-11-15T23:15:37,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/3172e9e9117e4568beb4e51c1bea1cc3 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/3172e9e9117e4568beb4e51c1bea1cc3 2024-11-15T23:15:37,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/69c9410024954107b29e53e3f8cd9cfa to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/69c9410024954107b29e53e3f8cd9cfa 2024-11-15T23:15:37,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/7ce04c434a95480e92b949596c314276 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/7ce04c434a95480e92b949596c314276 2024-11-15T23:15:37,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/fa4eec9e166542119aff1e0c011916bd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/fa4eec9e166542119aff1e0c011916bd 2024-11-15T23:15:37,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/a5b654005fe64a749c982ed270c5dd0d to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/a5b654005fe64a749c982ed270c5dd0d 2024-11-15T23:15:37,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/55f8de73703f4e57928c69fd0b811bad to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/55f8de73703f4e57928c69fd0b811bad 2024-11-15T23:15:37,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/53f1656164e64494be4ffb50e9ac8a7e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/53f1656164e64494be4ffb50e9ac8a7e 2024-11-15T23:15:37,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/5f4bef66d77e440bb2c77069b810a1dc to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/5f4bef66d77e440bb2c77069b810a1dc 2024-11-15T23:15:37,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/a16b1540936f49209823d28107269962 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/a16b1540936f49209823d28107269962 2024-11-15T23:15:37,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/fa9b2b7f64ed461cba8886a1282dab13 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/fa9b2b7f64ed461cba8886a1282dab13 2024-11-15T23:15:37,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/e0d6f3eb0449482aa72cf24b95378595 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/e0d6f3eb0449482aa72cf24b95378595 2024-11-15T23:15:37,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/df20849d661d492dbac2965450bbcbee to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/df20849d661d492dbac2965450bbcbee 2024-11-15T23:15:37,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/e6152bb5ee1c4bdc950d53734b91cdc0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/e6152bb5ee1c4bdc950d53734b91cdc0 2024-11-15T23:15:37,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/f50ae6fea47b40abbb215a8c8ae59dad to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/f50ae6fea47b40abbb215a8c8ae59dad 2024-11-15T23:15:37,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/c58540aaa3ec43caab33d697b5ae9531 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/c58540aaa3ec43caab33d697b5ae9531 2024-11-15T23:15:37,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/3e1dc691fdaa4286a13ab2d7a70e36d0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/3e1dc691fdaa4286a13ab2d7a70e36d0 2024-11-15T23:15:37,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/9d6698b8d9104bdabee3c5ec44ee0c67 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/9d6698b8d9104bdabee3c5ec44ee0c67 2024-11-15T23:15:37,127 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/recovered.edits/450.seqid, newMaxSeqId=450, maxSeqId=1 2024-11-15T23:15:37,128 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. 2024-11-15T23:15:37,129 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for b9ac2b510c393ceae30043c6a9eaa588: Waiting for close lock at 1731712535722Running coprocessor pre-close hooks at 1731712535722Disabling compacts and flushes for region at 1731712535722Disabling writes for close at 1731712535722Obtaining lock to block concurrent updates at 1731712535723 (+1 ms)Preparing flush snapshotting stores in b9ac2b510c393ceae30043c6a9eaa588 at 1731712535723Finished memstore snapshotting TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731712535724 (+1 ms)Flushing stores of TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588. at 1731712535725 (+1 ms)Flushing b9ac2b510c393ceae30043c6a9eaa588/A: creating writer at 1731712535725Flushing b9ac2b510c393ceae30043c6a9eaa588/A: appending metadata at 1731712535729 (+4 ms)Flushing b9ac2b510c393ceae30043c6a9eaa588/A: closing flushed file at 1731712535729Flushing b9ac2b510c393ceae30043c6a9eaa588/B: creating writer at 1731712536141 (+412 ms)Flushing b9ac2b510c393ceae30043c6a9eaa588/B: appending metadata at 1731712536144 (+3 ms)Flushing b9ac2b510c393ceae30043c6a9eaa588/B: closing flushed file at 1731712536144Flushing b9ac2b510c393ceae30043c6a9eaa588/C: creating writer at 1731712536559 (+415 ms)Flushing b9ac2b510c393ceae30043c6a9eaa588/C: appending metadata at 1731712536562 (+3 ms)Flushing b9ac2b510c393ceae30043c6a9eaa588/C: closing flushed file at 1731712536562Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9e9cb3e: reopening flushed file at 1731712536973 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68842dd4: reopening flushed file at 1731712536979 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45913aa0: reopening flushed file at 1731712536984 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for b9ac2b510c393ceae30043c6a9eaa588 in 1269ms, sequenceid=447, compaction requested=false at 1731712536991 (+7 ms)Writing region close event to WAL at 1731712537124 (+133 ms)Running coprocessor post-close hooks at 1731712537128 (+4 ms)Closed at 1731712537128 2024-11-15T23:15:37,131 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:37,131 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=b9ac2b510c393ceae30043c6a9eaa588, regionState=CLOSED 2024-11-15T23:15:37,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 because future has completed 2024-11-15T23:15:37,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-15T23:15:37,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure b9ac2b510c393ceae30043c6a9eaa588, server=d6ecb9b7dda1,38353,1731712503763 in 1.5730 sec 2024-11-15T23:15:37,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-15T23:15:37,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9ac2b510c393ceae30043c6a9eaa588, UNASSIGN in 1.5810 sec 2024-11-15T23:15:37,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-15T23:15:37,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5900 sec 2024-11-15T23:15:37,142 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712537142"}]},"ts":"1731712537142"} 2024-11-15T23:15:37,143 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T23:15:37,143 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T23:15:37,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6050 sec 2024-11-15T23:15:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T23:15:37,697 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:37,705 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T23:15:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:15:37,715 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:15:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T23:15:37,717 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:15:37,721 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:37,725 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A, FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B, FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C, FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/recovered.edits] 2024-11-15T23:15:37,728 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/5113b8131c8c438aa2ddc907aab8e055 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/5113b8131c8c438aa2ddc907aab8e055 2024-11-15T23:15:37,729 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/a93b50e1f4854cb8b7d12715e64db311 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/A/a93b50e1f4854cb8b7d12715e64db311 2024-11-15T23:15:37,731 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/42a8d43ae37d4ce49a9ef05b7e20ca71 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/42a8d43ae37d4ce49a9ef05b7e20ca71 2024-11-15T23:15:37,733 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/e00a213137c14bb9a554899aacc2aba3 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/B/e00a213137c14bb9a554899aacc2aba3 2024-11-15T23:15:37,735 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/2920b75032ab4a6fb8951f8328bb5509 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/2920b75032ab4a6fb8951f8328bb5509 2024-11-15T23:15:37,736 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/abc8e714a19047e3b5b98b7eeaa3d68c to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/C/abc8e714a19047e3b5b98b7eeaa3d68c 2024-11-15T23:15:37,738 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/recovered.edits/450.seqid to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588/recovered.edits/450.seqid 2024-11-15T23:15:37,738 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b9ac2b510c393ceae30043c6a9eaa588 2024-11-15T23:15:37,738 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T23:15:37,744 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:15:37,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-15T23:15:37,753 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T23:15:37,780 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T23:15:37,782 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:15:37,782 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T23:15:37,782 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731712537782"}]},"ts":"9223372036854775807"} 2024-11-15T23:15:37,786 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T23:15:37,786 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b9ac2b510c393ceae30043c6a9eaa588, NAME => 'TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T23:15:37,786 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T23:15:37,786 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731712537786"}]},"ts":"9223372036854775807"} 2024-11-15T23:15:37,789 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T23:15:37,790 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:15:37,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 84 msec 2024-11-15T23:15:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T23:15:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T23:15:37,826 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:37,838 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1925805657_22 at /127.0.0.1:49616 [Waiting for operation #111] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1603024157_22 at /127.0.0.1:32908 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1925805657_22 at /127.0.0.1:57086 [Waiting for operation #267] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1925805657_22 at /127.0.0.1:57224 [Waiting for operation #255] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1603024157_22 at /127.0.0.1:32900 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;d6ecb9b7dda1:38353-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1603024157_22 at /127.0.0.1:32888 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=467 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=436 (was 353) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5625 (was 6298) 2024-11-15T23:15:37,849 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=467, MaxFileDescriptor=1048576, SystemLoadAverage=436, ProcessCount=11, AvailableMemoryMB=5620 2024-11-15T23:15:37,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T23:15:37,850 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T23:15:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T23:15:37,853 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T23:15:37,853 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:37,853 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-15T23:15:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T23:15:37,855 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T23:15:37,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741942_1118 (size=1061) 2024-11-15T23:15:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T23:15:38,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T23:15:38,264 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444 2024-11-15T23:15:38,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741943_1119 (size=53) 2024-11-15T23:15:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T23:15:38,730 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:15:38,731 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b3f0f65ece63912349255b0111be7183, disabling compactions & flushes 2024-11-15T23:15:38,731 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:38,731 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:38,732 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. after waiting 0 ms 2024-11-15T23:15:38,732 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:38,732 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:38,732 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b3f0f65ece63912349255b0111be7183: Waiting for close lock at 1731712538731Disabling compacts and flushes for region at 1731712538731Disabling writes for close at 1731712538732 (+1 ms)Writing region close event to WAL at 1731712538732Closed at 1731712538732 2024-11-15T23:15:38,736 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T23:15:38,736 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731712538736"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731712538736"}]},"ts":"1731712538736"} 2024-11-15T23:15:38,739 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T23:15:38,740 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T23:15:38,741 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712538741"}]},"ts":"1731712538741"} 2024-11-15T23:15:38,742 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T23:15:38,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3f0f65ece63912349255b0111be7183, ASSIGN}] 2024-11-15T23:15:38,744 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3f0f65ece63912349255b0111be7183, ASSIGN 2024-11-15T23:15:38,745 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3f0f65ece63912349255b0111be7183, ASSIGN; state=OFFLINE, location=d6ecb9b7dda1,38353,1731712503763; forceNewPlan=false, retain=false 2024-11-15T23:15:38,896 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b3f0f65ece63912349255b0111be7183, regionState=OPENING, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:38,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3f0f65ece63912349255b0111be7183, ASSIGN because future has completed 2024-11-15T23:15:38,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763}] 2024-11-15T23:15:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T23:15:39,065 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:39,066 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} 2024-11-15T23:15:39,066 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:39,067 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:15:39,067 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:39,067 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:39,070 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:39,073 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:15:39,074 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3f0f65ece63912349255b0111be7183 columnFamilyName A 2024-11-15T23:15:39,074 DEBUG [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:39,075 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] regionserver.HStore(327): Store=b3f0f65ece63912349255b0111be7183/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:15:39,075 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:39,076 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:15:39,077 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3f0f65ece63912349255b0111be7183 columnFamilyName B 2024-11-15T23:15:39,077 DEBUG [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:39,077 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] regionserver.HStore(327): Store=b3f0f65ece63912349255b0111be7183/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:15:39,078 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:39,079 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:15:39,079 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3f0f65ece63912349255b0111be7183 columnFamilyName C 2024-11-15T23:15:39,079 DEBUG [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:39,080 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] regionserver.HStore(327): Store=b3f0f65ece63912349255b0111be7183/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:15:39,080 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:39,080 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:39,081 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:39,082 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:39,083 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:39,083 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:39,084 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T23:15:39,086 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:39,089 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T23:15:39,090 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened b3f0f65ece63912349255b0111be7183; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62742511, jitterRate=-0.0650637298822403}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T23:15:39,090 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:39,091 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for b3f0f65ece63912349255b0111be7183: Running coprocessor pre-open hook at 1731712539068Writing region info on filesystem at 1731712539068Initializing all the Stores at 1731712539069 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712539070 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712539070Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712539070Cleaning up temporary data from old regions at 1731712539083 (+13 ms)Running coprocessor post-open hooks at 1731712539090 (+7 ms)Region opened successfully at 1731712539091 (+1 ms) 2024-11-15T23:15:39,093 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., pid=30, masterSystemTime=1731712539056 2024-11-15T23:15:39,095 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:39,096 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:39,096 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b3f0f65ece63912349255b0111be7183, regionState=OPEN, openSeqNum=2, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:39,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 because future has completed 2024-11-15T23:15:39,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-15T23:15:39,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 in 198 msec 2024-11-15T23:15:39,103 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-15T23:15:39,103 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3f0f65ece63912349255b0111be7183, ASSIGN in 358 msec 2024-11-15T23:15:39,104 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T23:15:39,104 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712539104"}]},"ts":"1731712539104"} 2024-11-15T23:15:39,106 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T23:15:39,106 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T23:15:39,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2560 sec 2024-11-15T23:15:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T23:15:40,006 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:40,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bcb2494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:40,007 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:40,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:40,009 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:40,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:40,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:40,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@764a5610, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:40,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:40,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:40,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:40,012 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:40,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f9753ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:40,013 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:40,016 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:40,018 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T23:15:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is d6ecb9b7dda1,43421,1731712502855 2024-11-15T23:15:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@590288b4 2024-11-15T23:15:40,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T23:15:40,020 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T23:15:40,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T23:15:40,025 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T23:15:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T23:15:40,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741944_1120 (size=1097) 2024-11-15T23:15:40,449 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-15T23:15:40,449 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-15T23:15:40,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T23:15:40,463 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3f0f65ece63912349255b0111be7183, REOPEN/MOVE}] 2024-11-15T23:15:40,464 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3f0f65ece63912349255b0111be7183, REOPEN/MOVE 2024-11-15T23:15:40,465 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b3f0f65ece63912349255b0111be7183, regionState=CLOSING, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:40,467 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3f0f65ece63912349255b0111be7183, REOPEN/MOVE because future has completed 2024-11-15T23:15:40,468 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T23:15:40,468 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763}] 2024-11-15T23:15:40,620 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:40,620 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T23:15:40,620 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing b3f0f65ece63912349255b0111be7183, disabling compactions & flushes 2024-11-15T23:15:40,621 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:40,621 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:40,621 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. after waiting 0 ms 2024-11-15T23:15:40,621 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:40,624 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T23:15:40,625 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:40,625 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for b3f0f65ece63912349255b0111be7183: Waiting for close lock at 1731712540620Running coprocessor pre-close hooks at 1731712540620Disabling compacts and flushes for region at 1731712540620Disabling writes for close at 1731712540621 (+1 ms)Writing region close event to WAL at 1731712540621Running coprocessor post-close hooks at 1731712540625 (+4 ms)Closed at 1731712540625 2024-11-15T23:15:40,625 WARN [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: b3f0f65ece63912349255b0111be7183 to self. 2024-11-15T23:15:40,628 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:40,628 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b3f0f65ece63912349255b0111be7183, regionState=CLOSED 2024-11-15T23:15:40,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 because future has completed 2024-11-15T23:15:40,633 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-15T23:15:40,633 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 in 163 msec 2024-11-15T23:15:40,634 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3f0f65ece63912349255b0111be7183, REOPEN/MOVE; state=CLOSED, location=d6ecb9b7dda1,38353,1731712503763; forceNewPlan=false, retain=true 2024-11-15T23:15:40,785 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b3f0f65ece63912349255b0111be7183, regionState=OPENING, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:40,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3f0f65ece63912349255b0111be7183, REOPEN/MOVE because future has completed 2024-11-15T23:15:40,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763}] 2024-11-15T23:15:40,948 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:40,948 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} 2024-11-15T23:15:40,949 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:40,949 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:15:40,949 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:40,949 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:40,954 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:40,955 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:15:40,960 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3f0f65ece63912349255b0111be7183 columnFamilyName A 2024-11-15T23:15:40,962 DEBUG [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:40,963 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] regionserver.HStore(327): Store=b3f0f65ece63912349255b0111be7183/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:15:40,963 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:40,964 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:15:40,964 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3f0f65ece63912349255b0111be7183 columnFamilyName B 2024-11-15T23:15:40,964 DEBUG [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:40,965 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] regionserver.HStore(327): Store=b3f0f65ece63912349255b0111be7183/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:15:40,965 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:40,966 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:15:40,966 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3f0f65ece63912349255b0111be7183 columnFamilyName C 2024-11-15T23:15:40,966 DEBUG [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:40,966 INFO [StoreOpener-b3f0f65ece63912349255b0111be7183-1 {}] regionserver.HStore(327): Store=b3f0f65ece63912349255b0111be7183/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:15:40,966 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:40,967 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:40,967 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:40,968 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:40,969 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:40,969 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:40,970 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T23:15:40,971 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:40,972 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened b3f0f65ece63912349255b0111be7183; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68288547, jitterRate=0.017578646540641785}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T23:15:40,972 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:40,974 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for b3f0f65ece63912349255b0111be7183: Running coprocessor pre-open hook at 1731712540949Writing region info on filesystem at 1731712540949Initializing all the Stores at 1731712540951 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712540951Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712540951Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712540951Cleaning up temporary data from old regions at 1731712540969 (+18 ms)Running coprocessor post-open hooks at 1731712540972 (+3 ms)Region opened successfully at 1731712540974 (+2 ms) 2024-11-15T23:15:40,975 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., pid=35, masterSystemTime=1731712540944 2024-11-15T23:15:40,977 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:40,977 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:40,978 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b3f0f65ece63912349255b0111be7183, regionState=OPEN, openSeqNum=5, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:40,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 because future has completed 2024-11-15T23:15:40,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-15T23:15:40,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 in 190 msec 2024-11-15T23:15:40,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-15T23:15:40,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3f0f65ece63912349255b0111be7183, REOPEN/MOVE in 519 msec 2024-11-15T23:15:40,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-15T23:15:40,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 530 msec 2024-11-15T23:15:40,992 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 962 msec 2024-11-15T23:15:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-15T23:15:40,993 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:40,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:40,994 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:40,994 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:40,995 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbda171, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:40,997 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:40,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f90a6ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:40,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4406a4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:40,998 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:40,998 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:40,999 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442cd356, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:41,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:41,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:41,000 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:41,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5a397d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:41,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:41,003 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:41,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:41,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:41,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6628fbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:41,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:41,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:41,004 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:41,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd85eeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b236c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:41,006 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:41,007 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:41,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:41,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:41,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f395ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:41,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:41,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:41,008 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:41,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c528f97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26dcfede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,009 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:41,009 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:41,010 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ce4ccae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:41,012 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@711caa2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae3464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,013 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:41,013 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:41,014 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53269ece, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:41,016 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:41,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@708412a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbb11d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,017 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:41,017 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:41,018 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9e25ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:41,019 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:41,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dca7d23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2a6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,020 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:41,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:41,021 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:41,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:41,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:41,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f81573, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:41,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:41,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:41,022 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:41,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ef014ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ade04f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,024 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:15:41,024 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:15:41,024 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:15:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:15:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:15:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a70234b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:15:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:15:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:15:41,026 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:15:41,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62990a51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:15:41,028 DEBUG [Thread-620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:41,028 DEBUG [Thread-618 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:41,028 DEBUG [Thread-614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:41,028 DEBUG [Thread-629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:41,029 DEBUG [Thread-631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:41,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:41,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:41,029 DEBUG [Thread-629 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:41,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:41,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:41,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:41,029 DEBUG [Thread-629 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:41,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:41,030 DEBUG [Thread-631 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:41,030 DEBUG [Thread-631 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:41,030 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:41,030 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:41,030 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:41,031 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:41,031 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:41,032 DEBUG [Thread-625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:41,032 DEBUG [Thread-627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:41,034 DEBUG [Thread-616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:41,035 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:15:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:41,035 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:15:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-15T23:15:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:15:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:15:41,037 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:15:41,037 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:15:41,038 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:15:41,038 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:41,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:15:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T23:15:41,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:15:41,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:15:41,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:15:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:15:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:15:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:15:41,044 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:15:41,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:15:41,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:15:41,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:41,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T23:15:41,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:15:41,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:41,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:15:41,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:41,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:15:41,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:41,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T23:15:41,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111520d661ed2c484c6dabe1375f2b6fe5bb_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712541068/Put/seqid=0 2024-11-15T23:15:41,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712601166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712601169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712601177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50746 deadline: 1731712601177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712601179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741945_1121 (size=12154) 2024-11-15T23:15:41,190 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:41,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T23:15:41,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:41,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:41,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:41,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:41,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:41,199 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111520d661ed2c484c6dabe1375f2b6fe5bb_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111520d661ed2c484c6dabe1375f2b6fe5bb_b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:41,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:41,201 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/7d3c2ad948564cbca7321f77945ed1d4, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:41,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/7d3c2ad948564cbca7321f77945ed1d4 is 175, key is test_row_0/A:col10/1731712541068/Put/seqid=0 2024-11-15T23:15:41,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741946_1122 (size=30955) 2024-11-15T23:15:41,225 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/7d3c2ad948564cbca7321f77945ed1d4 2024-11-15T23:15:41,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/e05afa2d4b9847539849bdc8109a9c87 is 50, key is test_row_0/B:col10/1731712541068/Put/seqid=0 2024-11-15T23:15:41,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741947_1123 (size=12001) 2024-11-15T23:15:41,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/e05afa2d4b9847539849bdc8109a9c87 2024-11-15T23:15:41,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712601287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712601287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/bb61e4cc2a634ad7a39d4b80ac5d1d47 is 50, key is test_row_0/C:col10/1731712541068/Put/seqid=0 2024-11-15T23:15:41,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712601296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50746 deadline: 1731712601296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712601297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741948_1124 (size=12001) 2024-11-15T23:15:41,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/bb61e4cc2a634ad7a39d4b80ac5d1d47 2024-11-15T23:15:41,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/7d3c2ad948564cbca7321f77945ed1d4 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/7d3c2ad948564cbca7321f77945ed1d4 2024-11-15T23:15:41,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/7d3c2ad948564cbca7321f77945ed1d4, entries=150, sequenceid=16, filesize=30.2 K 2024-11-15T23:15:41,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/e05afa2d4b9847539849bdc8109a9c87 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/e05afa2d4b9847539849bdc8109a9c87 2024-11-15T23:15:41,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T23:15:41,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:41,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:41,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T23:15:41,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:41,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:41,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/e05afa2d4b9847539849bdc8109a9c87, entries=150, sequenceid=16, filesize=11.7 K 2024-11-15T23:15:41,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/bb61e4cc2a634ad7a39d4b80ac5d1d47 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/bb61e4cc2a634ad7a39d4b80ac5d1d47 2024-11-15T23:15:41,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/bb61e4cc2a634ad7a39d4b80ac5d1d47, entries=150, sequenceid=16, filesize=11.7 K 2024-11-15T23:15:41,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b3f0f65ece63912349255b0111be7183 in 257ms, sequenceid=16, compaction requested=false 2024-11-15T23:15:41,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:41,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712541163, max timestamp=1731712541496Num uniques 859; 2024-11-15T23:15:41,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:15:41,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712541163, max timestamp=1731712541496Num uniques 859; 2024-11-15T23:15:41,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:15:41,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T23:15:41,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:41,509 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=51.44 KB heapSize=144.25 KB 2024-11-15T23:15:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:41,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:41,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:15:41,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:41,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:15:41,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:41,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:15:41,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:41,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b0d1dde4cc434e5fa160a5f15774181f_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712541163/Put/seqid=0 2024-11-15T23:15:41,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741949_1125 (size=14594) 2024-11-15T23:15:41,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:41,558 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b0d1dde4cc434e5fa160a5f15774181f_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b0d1dde4cc434e5fa160a5f15774181f_b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:41,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/ff64021d15d94656bf11f7a187a3efe4, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:41,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/ff64021d15d94656bf11f7a187a3efe4 is 175, key is test_row_0/A:col10/1731712541163/Put/seqid=0 2024-11-15T23:15:41,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741950_1126 (size=39549) 2024-11-15T23:15:41,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50746 deadline: 1731712601585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712601585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731712541577, max timestamp=1731712541589Num uniques 859; 2024-11-15T23:15:41,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T23:15:41,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731712541577, max timestamp=1731712541589Num uniques 859; 2024-11-15T23:15:41,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T23:15:41,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731712541577, max timestamp=1731712541589Num uniques 859; 2024-11-15T23:15:41,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T23:15:41,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712601637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712601637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712601639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T23:15:41,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50746 deadline: 1731712601696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712601696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712601755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712601755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712601756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50746 deadline: 1731712601916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712601916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712601966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712601966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:41,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712601966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:41,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:41,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:41,975 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=38, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/ff64021d15d94656bf11f7a187a3efe4 2024-11-15T23:15:41,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/cd90d3bcff0d4e15bcef172f2bf626bb is 50, key is test_row_0/B:col10/1731712541163/Put/seqid=0 2024-11-15T23:15:42,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741951_1127 (size=12001) 2024-11-15T23:15:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T23:15:42,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50746 deadline: 1731712602225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:42,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:42,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712602225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:42,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:42,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:42,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:42,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712602285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:42,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712602285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:42,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712602285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:42,407 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/cd90d3bcff0d4e15bcef172f2bf626bb 2024-11-15T23:15:42,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/5de54f3301154706a51c5645bc59bba5 is 50, key is test_row_0/C:col10/1731712541163/Put/seqid=0 2024-11-15T23:15:42,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741952_1128 (size=12001) 2024-11-15T23:15:42,421 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/5de54f3301154706a51c5645bc59bba5 2024-11-15T23:15:42,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/ff64021d15d94656bf11f7a187a3efe4 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ff64021d15d94656bf11f7a187a3efe4 2024-11-15T23:15:42,430 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ff64021d15d94656bf11f7a187a3efe4, entries=200, sequenceid=38, filesize=38.6 K 2024-11-15T23:15:42,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/cd90d3bcff0d4e15bcef172f2bf626bb as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/cd90d3bcff0d4e15bcef172f2bf626bb 2024-11-15T23:15:42,435 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/cd90d3bcff0d4e15bcef172f2bf626bb, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T23:15:42,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/5de54f3301154706a51c5645bc59bba5 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/5de54f3301154706a51c5645bc59bba5 2024-11-15T23:15:42,442 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/5de54f3301154706a51c5645bc59bba5, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T23:15:42,443 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=120.76 KB/123660 for b3f0f65ece63912349255b0111be7183 in 935ms, sequenceid=38, compaction requested=false 2024-11-15T23:15:42,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:42,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:42,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-15T23:15:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-15T23:15:42,452 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-15T23:15:42,452 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4030 sec 2024-11-15T23:15:42,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.4180 sec 2024-11-15T23:15:42,669 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T23:15:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:42,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-15T23:15:42,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:15:42,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:42,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:15:42,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:42,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:15:42,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:42,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e65bd466c5114efe87a4afcce03258ad_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712542736/Put/seqid=0 2024-11-15T23:15:42,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741953_1129 (size=17034) 2024-11-15T23:15:42,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712602768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:42,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:42,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50746 deadline: 1731712602773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:42,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:42,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:42,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712602795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:42,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:42,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712602795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712602795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:42,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712602875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:42,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50746 deadline: 1731712602895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:42,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:42,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:43,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712603085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:43,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50746 deadline: 1731712603105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:43,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:43,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:43,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:43,163 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:43,170 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e65bd466c5114efe87a4afcce03258ad_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e65bd466c5114efe87a4afcce03258ad_b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:43,171 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/6e82ffdc266e4f6fa5e9053a7b71dc15, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:43,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/6e82ffdc266e4f6fa5e9053a7b71dc15 is 175, key is test_row_0/A:col10/1731712542736/Put/seqid=0 2024-11-15T23:15:43,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741954_1130 (size=48139) 2024-11-15T23:15:43,185 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=73, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/6e82ffdc266e4f6fa5e9053a7b71dc15 2024-11-15T23:15:43,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T23:15:43,186 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:43,186 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:15:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-15T23:15:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T23:15:43,189 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:15:43,190 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:15:43,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:15:43,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/140fc76b84d847aca6a3413545a9b16f is 50, key is test_row_0/B:col10/1731712542736/Put/seqid=0 2024-11-15T23:15:43,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741955_1131 (size=12001) 2024-11-15T23:15:43,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/140fc76b84d847aca6a3413545a9b16f 2024-11-15T23:15:43,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/24a9207601ca4d06b3f708e3baddcb38 is 50, key is test_row_0/C:col10/1731712542736/Put/seqid=0 2024-11-15T23:15:43,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741956_1132 (size=12001) 2024-11-15T23:15:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T23:15:43,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T23:15:43,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:43,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:43,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:43,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:43,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:43,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:43,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712603395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:43,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:43,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50746 deadline: 1731712603425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:43,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T23:15:43,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:43,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:43,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:43,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:43,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T23:15:43,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/24a9207601ca4d06b3f708e3baddcb38 2024-11-15T23:15:43,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/6e82ffdc266e4f6fa5e9053a7b71dc15 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/6e82ffdc266e4f6fa5e9053a7b71dc15 2024-11-15T23:15:43,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T23:15:43,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:43,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:43,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:43,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:43,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:43,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/6e82ffdc266e4f6fa5e9053a7b71dc15, entries=250, sequenceid=73, filesize=47.0 K 2024-11-15T23:15:43,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/140fc76b84d847aca6a3413545a9b16f as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/140fc76b84d847aca6a3413545a9b16f 2024-11-15T23:15:43,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/140fc76b84d847aca6a3413545a9b16f, entries=150, sequenceid=73, filesize=11.7 K 2024-11-15T23:15:43,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/24a9207601ca4d06b3f708e3baddcb38 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/24a9207601ca4d06b3f708e3baddcb38 2024-11-15T23:15:43,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/24a9207601ca4d06b3f708e3baddcb38, entries=150, sequenceid=73, filesize=11.7 K 2024-11-15T23:15:43,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for b3f0f65ece63912349255b0111be7183 in 929ms, sequenceid=73, compaction requested=true 2024-11-15T23:15:43,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:43,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:15:43,667 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:43,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:43,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:15:43,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:43,667 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:43,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:15:43,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:43,669 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:43,669 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:43,669 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/B is initiating minor compaction (all files) 2024-11-15T23:15:43,669 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/A is initiating minor compaction (all files) 2024-11-15T23:15:43,669 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/A in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:43,669 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/B in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:43,669 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/e05afa2d4b9847539849bdc8109a9c87, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/cd90d3bcff0d4e15bcef172f2bf626bb, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/140fc76b84d847aca6a3413545a9b16f] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=35.2 K 2024-11-15T23:15:43,669 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/7d3c2ad948564cbca7321f77945ed1d4, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ff64021d15d94656bf11f7a187a3efe4, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/6e82ffdc266e4f6fa5e9053a7b71dc15] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=115.9 K 2024-11-15T23:15:43,669 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:43,670 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. files: [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/7d3c2ad948564cbca7321f77945ed1d4, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ff64021d15d94656bf11f7a187a3efe4, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/6e82ffdc266e4f6fa5e9053a7b71dc15] 2024-11-15T23:15:43,670 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting e05afa2d4b9847539849bdc8109a9c87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731712541068 2024-11-15T23:15:43,670 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d3c2ad948564cbca7321f77945ed1d4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731712541068 2024-11-15T23:15:43,670 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting cd90d3bcff0d4e15bcef172f2bf626bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731712541163 2024-11-15T23:15:43,670 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff64021d15d94656bf11f7a187a3efe4, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731712541163 2024-11-15T23:15:43,671 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 140fc76b84d847aca6a3413545a9b16f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731712541616 2024-11-15T23:15:43,671 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e82ffdc266e4f6fa5e9053a7b71dc15, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731712541616 2024-11-15T23:15:43,691 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#B#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:43,692 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/77fa0cb42caa45c6a3ba9371d2318bcb is 50, key is test_row_0/B:col10/1731712542736/Put/seqid=0 2024-11-15T23:15:43,694 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:43,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741957_1133 (size=12104) 2024-11-15T23:15:43,700 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115c5f6c4c95cf9472783a24fbf458f08a9_b3f0f65ece63912349255b0111be7183 store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:43,706 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115c5f6c4c95cf9472783a24fbf458f08a9_b3f0f65ece63912349255b0111be7183, store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:43,706 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c5f6c4c95cf9472783a24fbf458f08a9_b3f0f65ece63912349255b0111be7183 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:43,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741958_1134 (size=4469) 2024-11-15T23:15:43,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T23:15:43,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:43,804 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T23:15:43,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:15:43,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:43,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:15:43,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:43,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:15:43,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:43,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c24f70a5c34b49cbb7048b2350ce9953_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712542763/Put/seqid=0 2024-11-15T23:15:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T23:15:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:43,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:43,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741959_1135 (size=12154) 2024-11-15T23:15:43,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:43,833 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c24f70a5c34b49cbb7048b2350ce9953_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c24f70a5c34b49cbb7048b2350ce9953_b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:43,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/9d85301e07e14a5e86e2fde43ce5f6da, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:43,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/9d85301e07e14a5e86e2fde43ce5f6da is 175, key is test_row_0/A:col10/1731712542763/Put/seqid=0 2024-11-15T23:15:43,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741960_1136 (size=30955) 2024-11-15T23:15:43,841 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/9d85301e07e14a5e86e2fde43ce5f6da 2024-11-15T23:15:43,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/a01bbb26a9894547a1804022331bd91e is 50, key is test_row_0/B:col10/1731712542763/Put/seqid=0 2024-11-15T23:15:43,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741961_1137 (size=12001) 2024-11-15T23:15:43,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731712543850, max timestamp=1731712543878Num uniques 859; 2024-11-15T23:15:43,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731712543850, max timestamp=1731712543878Num uniques 859; 2024-11-15T23:15:43,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731712543850, max timestamp=1731712543878Num uniques 859; 2024-11-15T23:15:43,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T23:15:43,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T23:15:43,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T23:15:43,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712603910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:43,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712603910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:43,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712603912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:43,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712603919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:43,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:43,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:43,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:43,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50746 deadline: 1731712603935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:43,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:43,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:43,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:44,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712604026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:44,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712604026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:44,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:44,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712604026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:44,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:44,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712604035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:44,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:44,106 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/77fa0cb42caa45c6a3ba9371d2318bcb as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/77fa0cb42caa45c6a3ba9371d2318bcb 2024-11-15T23:15:44,114 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3f0f65ece63912349255b0111be7183/B of b3f0f65ece63912349255b0111be7183 into 77fa0cb42caa45c6a3ba9371d2318bcb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:44,114 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:44,114 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/B, priority=13, startTime=1731712543667; duration=0sec 2024-11-15T23:15:44,114 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:44,114 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:B 2024-11-15T23:15:44,114 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:44,115 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:44,115 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/C is initiating minor compaction (all files) 2024-11-15T23:15:44,116 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/C in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:44,116 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/bb61e4cc2a634ad7a39d4b80ac5d1d47, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/5de54f3301154706a51c5645bc59bba5, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/24a9207601ca4d06b3f708e3baddcb38] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=35.2 K 2024-11-15T23:15:44,117 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting bb61e4cc2a634ad7a39d4b80ac5d1d47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731712541068 2024-11-15T23:15:44,118 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 5de54f3301154706a51c5645bc59bba5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731712541163 2024-11-15T23:15:44,118 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 24a9207601ca4d06b3f708e3baddcb38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731712541616 2024-11-15T23:15:44,125 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#A#compaction#115 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:44,126 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/ea99a96f7ea042cf9568451a86d4dc8a is 175, key is test_row_0/A:col10/1731712542736/Put/seqid=0 2024-11-15T23:15:44,130 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#C#compaction#118 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:44,131 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/ffa7752c5ead41c3967263cc8c62de28 is 50, key is test_row_0/C:col10/1731712542736/Put/seqid=0 2024-11-15T23:15:44,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741962_1138 (size=31058) 2024-11-15T23:15:44,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741963_1139 (size=12104) 2024-11-15T23:15:44,147 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/ea99a96f7ea042cf9568451a86d4dc8a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ea99a96f7ea042cf9568451a86d4dc8a 2024-11-15T23:15:44,157 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3f0f65ece63912349255b0111be7183/A of b3f0f65ece63912349255b0111be7183 into ea99a96f7ea042cf9568451a86d4dc8a(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:44,157 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:44,157 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/A, priority=13, startTime=1731712543667; duration=0sec 2024-11-15T23:15:44,157 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:44,157 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:A 2024-11-15T23:15:44,157 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/ffa7752c5ead41c3967263cc8c62de28 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/ffa7752c5ead41c3967263cc8c62de28 2024-11-15T23:15:44,166 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3f0f65ece63912349255b0111be7183/C of b3f0f65ece63912349255b0111be7183 into ffa7752c5ead41c3967263cc8c62de28(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:44,166 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:44,167 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/C, priority=13, startTime=1731712543667; duration=0sec 2024-11-15T23:15:44,167 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:44,167 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:C 2024-11-15T23:15:44,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712604236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:44,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:44,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712604237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:44,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:44,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712604237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:44,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712604245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:44,270 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/a01bbb26a9894547a1804022331bd91e 2024-11-15T23:15:44,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/1791ac203daf47039e4cac355046566d is 50, key is test_row_0/C:col10/1731712542763/Put/seqid=0 2024-11-15T23:15:44,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741964_1140 (size=12001) 2024-11-15T23:15:44,286 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/1791ac203daf47039e4cac355046566d 2024-11-15T23:15:44,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/9d85301e07e14a5e86e2fde43ce5f6da as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/9d85301e07e14a5e86e2fde43ce5f6da 2024-11-15T23:15:44,297 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/9d85301e07e14a5e86e2fde43ce5f6da, entries=150, sequenceid=83, filesize=30.2 K 2024-11-15T23:15:44,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/a01bbb26a9894547a1804022331bd91e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/a01bbb26a9894547a1804022331bd91e 2024-11-15T23:15:44,306 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/a01bbb26a9894547a1804022331bd91e, entries=150, sequenceid=83, filesize=11.7 K 2024-11-15T23:15:44,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/1791ac203daf47039e4cac355046566d as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/1791ac203daf47039e4cac355046566d 2024-11-15T23:15:44,313 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/1791ac203daf47039e4cac355046566d, entries=150, sequenceid=83, filesize=11.7 K 2024-11-15T23:15:44,314 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for b3f0f65ece63912349255b0111be7183 in 510ms, sequenceid=83, compaction requested=false 2024-11-15T23:15:44,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:44,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:44,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-15T23:15:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-15T23:15:44,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-15T23:15:44,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1260 sec 2024-11-15T23:15:44,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.1320 sec 2024-11-15T23:15:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T23:15:44,326 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:44,326 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:15:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-15T23:15:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T23:15:44,328 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:15:44,328 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:15:44,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:15:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T23:15:44,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T23:15:44,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:44,481 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-15T23:15:44,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:15:44,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:44,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:15:44,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:44,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:15:44,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:44,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150949da2153f34eabb1d0dd98e45332cb_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712543916/Put/seqid=0 2024-11-15T23:15:44,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741965_1141 (size=12154) 2024-11-15T23:15:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:44,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:44,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712604555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:44,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712604555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:44,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712604556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:44,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712604565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:44,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T23:15:44,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712604675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:44,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712604675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712604675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:44,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:44,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:44,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712604885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:44,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712604885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:44,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712604885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:44,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:44,896 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150949da2153f34eabb1d0dd98e45332cb_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150949da2153f34eabb1d0dd98e45332cb_b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:44,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/49eb0aa43a8d4240b7cdf718773d3781, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:44,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/49eb0aa43a8d4240b7cdf718773d3781 is 175, key is test_row_0/A:col10/1731712543916/Put/seqid=0 2024-11-15T23:15:44,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741966_1142 (size=30955) 2024-11-15T23:15:44,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T23:15:44,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:44,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50746 deadline: 1731712604955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:44,958 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in b3f0f65ece63912349255b0111be7183 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:15:44,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:44,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:45,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:45,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712605075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:45,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:45,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:45,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:45,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712605195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:45,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:45,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712605195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:45,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:45,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:45,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712605195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:45,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:45,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:45,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:45,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:45,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:45,302 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/49eb0aa43a8d4240b7cdf718773d3781 2024-11-15T23:15:45,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/3cf3b4848e98440a901094a233790239 is 50, key is test_row_0/B:col10/1731712543916/Put/seqid=0 2024-11-15T23:15:45,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741967_1143 (size=12001) 2024-11-15T23:15:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T23:15:45,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712605705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:45,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:45,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712605706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712605706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:45,723 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/3cf3b4848e98440a901094a233790239 2024-11-15T23:15:45,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/d8abdfb2eba0418eb1ec6bfce465f158 is 50, key is test_row_0/C:col10/1731712543916/Put/seqid=0 2024-11-15T23:15:45,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741968_1144 (size=12001) 2024-11-15T23:15:46,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712606085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:46,092 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in b3f0f65ece63912349255b0111be7183 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:15:46,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:46,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:46,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:46,138 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/d8abdfb2eba0418eb1ec6bfce465f158 2024-11-15T23:15:46,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/49eb0aa43a8d4240b7cdf718773d3781 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/49eb0aa43a8d4240b7cdf718773d3781 2024-11-15T23:15:46,148 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/49eb0aa43a8d4240b7cdf718773d3781, entries=150, sequenceid=121, filesize=30.2 K 2024-11-15T23:15:46,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/3cf3b4848e98440a901094a233790239 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/3cf3b4848e98440a901094a233790239 2024-11-15T23:15:46,153 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/3cf3b4848e98440a901094a233790239, entries=150, sequenceid=121, filesize=11.7 K 2024-11-15T23:15:46,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/d8abdfb2eba0418eb1ec6bfce465f158 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/d8abdfb2eba0418eb1ec6bfce465f158 2024-11-15T23:15:46,160 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/d8abdfb2eba0418eb1ec6bfce465f158, entries=150, sequenceid=121, filesize=11.7 K 2024-11-15T23:15:46,161 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for b3f0f65ece63912349255b0111be7183 in 1679ms, sequenceid=121, compaction requested=true 2024-11-15T23:15:46,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:46,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:46,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-15T23:15:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-15T23:15:46,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-15T23:15:46,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8350 sec 2024-11-15T23:15:46,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.8400 sec 2024-11-15T23:15:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T23:15:46,475 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:46,476 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:15:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-15T23:15:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T23:15:46,477 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:15:46,480 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:15:46,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:15:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T23:15:46,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T23:15:46,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:46,633 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T23:15:46,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:15:46,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:46,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:15:46,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:46,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:15:46,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:46,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111560952cf0b3f34a4a8ffd800318f14cfd_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712544563/Put/seqid=0 2024-11-15T23:15:46,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741969_1145 (size=12254) 2024-11-15T23:15:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:46,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:46,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731712546735, max timestamp=1731712546753Num uniques 859; 2024-11-15T23:15:46,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731712546735, max timestamp=1731712546753Num uniques 859; 2024-11-15T23:15:46,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-15T23:15:46,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-15T23:15:46,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731712546735, max timestamp=1731712546753Num uniques 859; 2024-11-15T23:15:46,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-15T23:15:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T23:15:46,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712606792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:46,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:46,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:46,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:46,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712606792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:46,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:46,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:46,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712606793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:46,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:46,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:46,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:46,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:46,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:46,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712606906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712606906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:46,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712606906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:46,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50746 deadline: 1731712606965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:46,972 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in b3f0f65ece63912349255b0111be7183 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:15:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:47,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:47,050 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111560952cf0b3f34a4a8ffd800318f14cfd_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111560952cf0b3f34a4a8ffd800318f14cfd_b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:47,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/49d64ca5d11e4b7490347b5aafd635c6, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:47,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/49d64ca5d11e4b7490347b5aafd635c6 is 175, key is test_row_0/A:col10/1731712544563/Put/seqid=0 2024-11-15T23:15:47,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741970_1146 (size=31055) 2024-11-15T23:15:47,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T23:15:47,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712607115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:47,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712607115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:47,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712607115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:47,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712607425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:47,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712607425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:47,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712607425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:47,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:47,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:47,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:47,456 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/49d64ca5d11e4b7490347b5aafd635c6 2024-11-15T23:15:47,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/1d90c0b0aae14c658071ef2c7e2f06c2 is 50, key is test_row_0/B:col10/1731712544563/Put/seqid=0 2024-11-15T23:15:47,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741971_1147 (size=12101) 2024-11-15T23:15:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T23:15:47,869 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/1d90c0b0aae14c658071ef2c7e2f06c2 2024-11-15T23:15:47,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/8d4d2df8791549ccbdd96a988028a839 is 50, key is test_row_0/C:col10/1731712544563/Put/seqid=0 2024-11-15T23:15:47,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741972_1148 (size=12101) 2024-11-15T23:15:47,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712607935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:47,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712607935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:47,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712607935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:47,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:48,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:48,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712608106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:48,110 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in b3f0f65ece63912349255b0111be7183 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:15:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:48,281 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/8d4d2df8791549ccbdd96a988028a839 2024-11-15T23:15:48,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/49d64ca5d11e4b7490347b5aafd635c6 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/49d64ca5d11e4b7490347b5aafd635c6 2024-11-15T23:15:48,295 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/49d64ca5d11e4b7490347b5aafd635c6, entries=150, sequenceid=131, filesize=30.3 K 2024-11-15T23:15:48,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/1d90c0b0aae14c658071ef2c7e2f06c2 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/1d90c0b0aae14c658071ef2c7e2f06c2 2024-11-15T23:15:48,301 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/1d90c0b0aae14c658071ef2c7e2f06c2, entries=150, sequenceid=131, filesize=11.8 K 2024-11-15T23:15:48,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/8d4d2df8791549ccbdd96a988028a839 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/8d4d2df8791549ccbdd96a988028a839 2024-11-15T23:15:48,306 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/8d4d2df8791549ccbdd96a988028a839, entries=150, sequenceid=131, filesize=11.8 K 2024-11-15T23:15:48,307 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for b3f0f65ece63912349255b0111be7183 in 1675ms, sequenceid=131, compaction requested=true 2024-11-15T23:15:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-15T23:15:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-15T23:15:48,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-15T23:15:48,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8280 sec 2024-11-15T23:15:48,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.8350 sec 2024-11-15T23:15:48,440 INFO [master/d6ecb9b7dda1:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-15T23:15:48,440 INFO [master/d6ecb9b7dda1:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-15T23:15:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T23:15:48,626 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:48,626 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:15:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-15T23:15:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T23:15:48,628 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:15:48,629 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:15:48,629 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:15:48,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T23:15:48,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T23:15:48,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:48,782 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-15T23:15:48,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:15:48,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:48,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:15:48,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:48,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:15:48,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:48,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111577450401c6644509a2f3a561d9206b5f_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712546788/Put/seqid=0 2024-11-15T23:15:48,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741973_1149 (size=12304) 2024-11-15T23:15:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T23:15:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:48,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:48,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712608955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:48,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712608956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:48,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712608960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:48,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:48,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:48,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:49,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:49,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712609065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:49,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712609065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:49,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712609075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:49,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:49,205 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111577450401c6644509a2f3a561d9206b5f_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111577450401c6644509a2f3a561d9206b5f_b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:49,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/cdc2239ef96d4759b5201adfafd749c7, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:49,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/cdc2239ef96d4759b5201adfafd749c7 is 175, key is test_row_0/A:col10/1731712546788/Put/seqid=0 2024-11-15T23:15:49,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741974_1150 (size=31105) 2024-11-15T23:15:49,220 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/cdc2239ef96d4759b5201adfafd749c7 2024-11-15T23:15:49,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/5b41f246bcb34a9ea24bf587f519b0f7 is 50, key is test_row_0/B:col10/1731712546788/Put/seqid=0 2024-11-15T23:15:49,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741975_1151 (size=12151) 2024-11-15T23:15:49,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T23:15:49,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712609276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:49,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:49,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712609276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:49,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:49,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712609285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:49,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:49,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:49,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:49,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712609585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:49,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:49,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:49,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712609585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:49,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:49,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:49,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:49,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:49,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712609595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:49,635 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/5b41f246bcb34a9ea24bf587f519b0f7 2024-11-15T23:15:49,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/4497c099f8294e379d181149a56398e8 is 50, key is test_row_0/C:col10/1731712546788/Put/seqid=0 2024-11-15T23:15:49,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741976_1152 (size=12151) 2024-11-15T23:15:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T23:15:50,049 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/4497c099f8294e379d181149a56398e8 2024-11-15T23:15:50,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/cdc2239ef96d4759b5201adfafd749c7 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/cdc2239ef96d4759b5201adfafd749c7 2024-11-15T23:15:50,063 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/cdc2239ef96d4759b5201adfafd749c7, entries=150, sequenceid=165, filesize=30.4 K 2024-11-15T23:15:50,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/5b41f246bcb34a9ea24bf587f519b0f7 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/5b41f246bcb34a9ea24bf587f519b0f7 2024-11-15T23:15:50,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,080 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/5b41f246bcb34a9ea24bf587f519b0f7, entries=150, sequenceid=165, filesize=11.9 K 2024-11-15T23:15:50,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/4497c099f8294e379d181149a56398e8 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/4497c099f8294e379d181149a56398e8 2024-11-15T23:15:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,107 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/4497c099f8294e379d181149a56398e8, entries=150, sequenceid=165, filesize=11.9 K 2024-11-15T23:15:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,110 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for b3f0f65ece63912349255b0111be7183 in 1329ms, sequenceid=165, compaction requested=true 2024-11-15T23:15:50,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:50,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:50,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-15T23:15:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-15T23:15:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-15T23:15:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4830 sec 2024-11-15T23:15:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4910 sec 2024-11-15T23:15:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T23:15:50,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:15:50,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:50,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:15:50,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:50,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:15:50,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ef719ce087824e7ebf3ccae736b8b5e3_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_1/A:col10/1731712550144/Put/seqid=0 2024-11-15T23:15:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741977_1153 (size=12304) 2024-11-15T23:15:50,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731712550203, max timestamp=1731712550250Num uniques 859; 2024-11-15T23:15:50,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-15T23:15:50,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731712550234, max timestamp=1731712550250Num uniques 859; 2024-11-15T23:15:50,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T23:15:50,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731712550234, max timestamp=1731712550250Num uniques 859; 2024-11-15T23:15:50,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T23:15:50,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:50,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712610286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:50,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:50,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:50,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712610287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:50,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:50,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:50,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712610289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:50,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:50,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712610395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:50,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712610396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:50,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712610396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:50,594 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:50,598 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ef719ce087824e7ebf3ccae736b8b5e3_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ef719ce087824e7ebf3ccae736b8b5e3_b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:50,599 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/6ba81febd8a84fa380412b523495b25a, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:50,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/6ba81febd8a84fa380412b523495b25a is 175, key is test_row_1/A:col10/1731712550144/Put/seqid=0 2024-11-15T23:15:50,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741978_1154 (size=31101) 2024-11-15T23:15:50,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:50,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712610615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:50,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712610616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712610616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T23:15:50,785 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:50,786 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:15:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-15T23:15:50,787 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:15:50,788 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:15:50,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:15:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T23:15:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T23:15:50,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712610925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:50,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:50,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712610936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:50,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:50,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712610936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:50,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:50,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:50,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:50,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T23:15:50,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:50,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:50,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:50,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:50,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:51,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50746 deadline: 1731712610995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:51,000 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in b3f0f65ece63912349255b0111be7183 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:15:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:51,004 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/6ba81febd8a84fa380412b523495b25a 2024-11-15T23:15:51,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/b0bfc55e45f74a3e8f4e147b5ba79513 is 50, key is test_row_1/B:col10/1731712550144/Put/seqid=0 2024-11-15T23:15:51,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741979_1155 (size=9757) 2024-11-15T23:15:51,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T23:15:51,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:51,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:51,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:51,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:51,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T23:15:51,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T23:15:51,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:51,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:51,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:51,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:51,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:51,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T23:15:51,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:51,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:51,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:51,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:51,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T23:15:51,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/b0bfc55e45f74a3e8f4e147b5ba79513 2024-11-15T23:15:51,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712611435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:51,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:51,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:51,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:51,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/b601a29d07b34103a20028b53bce1282 is 50, key is test_row_1/C:col10/1731712550144/Put/seqid=0 2024-11-15T23:15:51,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741980_1156 (size=9757) 2024-11-15T23:15:51,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712611445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:51,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:51,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:51,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712611445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:51,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:51,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:51,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:51,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:51,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T23:15:51,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:51,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:51,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:51,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:51,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:51,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T23:15:51,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:51,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:51,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:51,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:51,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:51,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/b601a29d07b34103a20028b53bce1282 2024-11-15T23:15:51,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/6ba81febd8a84fa380412b523495b25a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/6ba81febd8a84fa380412b523495b25a 2024-11-15T23:15:51,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/6ba81febd8a84fa380412b523495b25a, entries=150, sequenceid=176, filesize=30.4 K 2024-11-15T23:15:51,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/b0bfc55e45f74a3e8f4e147b5ba79513 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/b0bfc55e45f74a3e8f4e147b5ba79513 2024-11-15T23:15:51,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/b0bfc55e45f74a3e8f4e147b5ba79513, entries=100, sequenceid=176, filesize=9.5 K 2024-11-15T23:15:51,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T23:15:51,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:51,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:51,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:51,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:51,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:51,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/b601a29d07b34103a20028b53bce1282 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/b601a29d07b34103a20028b53bce1282 2024-11-15T23:15:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:51,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/b601a29d07b34103a20028b53bce1282, entries=100, sequenceid=176, filesize=9.5 K 2024-11-15T23:15:51,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.94 KB/135110 for b3f0f65ece63912349255b0111be7183 in 1722ms, sequenceid=176, compaction requested=true 2024-11-15T23:15:51,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:51,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:15:51,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:51,869 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-15T23:15:51,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:15:51,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:51,869 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-15T23:15:51,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:15:51,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:51,871 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 186229 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-15T23:15:51,871 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/A is initiating minor compaction (all files) 2024-11-15T23:15:51,871 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/A in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:51,871 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ea99a96f7ea042cf9568451a86d4dc8a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/9d85301e07e14a5e86e2fde43ce5f6da, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/49eb0aa43a8d4240b7cdf718773d3781, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/49d64ca5d11e4b7490347b5aafd635c6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/cdc2239ef96d4759b5201adfafd749c7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/6ba81febd8a84fa380412b523495b25a] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=181.9 K 2024-11-15T23:15:51,871 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:51,871 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 70115 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-15T23:15:51,871 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. files: [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ea99a96f7ea042cf9568451a86d4dc8a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/9d85301e07e14a5e86e2fde43ce5f6da, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/49eb0aa43a8d4240b7cdf718773d3781, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/49d64ca5d11e4b7490347b5aafd635c6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/cdc2239ef96d4759b5201adfafd749c7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/6ba81febd8a84fa380412b523495b25a] 2024-11-15T23:15:51,871 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/B is initiating minor compaction (all files) 2024-11-15T23:15:51,871 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/B in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:51,871 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/77fa0cb42caa45c6a3ba9371d2318bcb, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/a01bbb26a9894547a1804022331bd91e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/3cf3b4848e98440a901094a233790239, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/1d90c0b0aae14c658071ef2c7e2f06c2, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/5b41f246bcb34a9ea24bf587f519b0f7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/b0bfc55e45f74a3e8f4e147b5ba79513] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=68.5 K 2024-11-15T23:15:51,872 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea99a96f7ea042cf9568451a86d4dc8a, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731712541616 2024-11-15T23:15:51,872 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 77fa0cb42caa45c6a3ba9371d2318bcb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731712541616 2024-11-15T23:15:51,872 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d85301e07e14a5e86e2fde43ce5f6da, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731712542758 2024-11-15T23:15:51,872 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting a01bbb26a9894547a1804022331bd91e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731712542758 2024-11-15T23:15:51,873 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49eb0aa43a8d4240b7cdf718773d3781, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731712543906 2024-11-15T23:15:51,873 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cf3b4848e98440a901094a233790239, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731712543906 2024-11-15T23:15:51,873 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49d64ca5d11e4b7490347b5aafd635c6, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731712544552 2024-11-15T23:15:51,873 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d90c0b0aae14c658071ef2c7e2f06c2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731712544552 2024-11-15T23:15:51,873 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdc2239ef96d4759b5201adfafd749c7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731712546784 2024-11-15T23:15:51,873 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b41f246bcb34a9ea24bf587f519b0f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731712546784 2024-11-15T23:15:51,874 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ba81febd8a84fa380412b523495b25a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731712548955 2024-11-15T23:15:51,874 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting b0bfc55e45f74a3e8f4e147b5ba79513, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731712548955 2024-11-15T23:15:51,888 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:51,894 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#B#compaction#133 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:51,895 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/2fcf0bf87b824156be0bbca63f049600 is 50, key is test_row_0/B:col10/1731712546788/Put/seqid=0 2024-11-15T23:15:51,909 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115ce71498bd2ba460681ac84752311f55c_b3f0f65ece63912349255b0111be7183 store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:51,912 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115ce71498bd2ba460681ac84752311f55c_b3f0f65ece63912349255b0111be7183, store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:51,913 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ce71498bd2ba460681ac84752311f55c_b3f0f65ece63912349255b0111be7183 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T23:15:51,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741982_1158 (size=4469) 2024-11-15T23:15:51,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741981_1157 (size=12459) 2024-11-15T23:15:51,944 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/2fcf0bf87b824156be0bbca63f049600 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/2fcf0bf87b824156be0bbca63f049600 2024-11-15T23:15:51,949 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in b3f0f65ece63912349255b0111be7183/B of b3f0f65ece63912349255b0111be7183 into 2fcf0bf87b824156be0bbca63f049600(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:51,949 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:51,949 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/B, priority=10, startTime=1731712551869; duration=0sec 2024-11-15T23:15:51,949 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:51,949 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:B 2024-11-15T23:15:51,949 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-15T23:15:51,952 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 70115 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-15T23:15:51,952 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/C is initiating minor compaction (all files) 2024-11-15T23:15:51,952 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/C in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:51,952 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/ffa7752c5ead41c3967263cc8c62de28, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/1791ac203daf47039e4cac355046566d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/d8abdfb2eba0418eb1ec6bfce465f158, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/8d4d2df8791549ccbdd96a988028a839, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/4497c099f8294e379d181149a56398e8, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/b601a29d07b34103a20028b53bce1282] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=68.5 K 2024-11-15T23:15:51,953 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting ffa7752c5ead41c3967263cc8c62de28, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731712541616 2024-11-15T23:15:51,953 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 1791ac203daf47039e4cac355046566d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731712542758 2024-11-15T23:15:51,953 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting d8abdfb2eba0418eb1ec6bfce465f158, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731712543906 2024-11-15T23:15:51,953 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d4d2df8791549ccbdd96a988028a839, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731712544552 2024-11-15T23:15:51,954 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 4497c099f8294e379d181149a56398e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731712546784 2024-11-15T23:15:51,954 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting b601a29d07b34103a20028b53bce1282, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731712548955 2024-11-15T23:15:51,970 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#C#compaction#134 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:51,971 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/19eceed266f14fa7b6492665f8eb4f04 is 50, key is test_row_0/C:col10/1731712546788/Put/seqid=0 2024-11-15T23:15:52,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741983_1159 (size=12459) 2024-11-15T23:15:52,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T23:15:52,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:52,017 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=131.94 KB heapSize=365.84 KB 2024-11-15T23:15:52,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:15:52,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:52,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:15:52,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:52,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:15:52,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:15:52,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b4692d79f44b415eaed7497479d7bae9_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712550287/Put/seqid=0 2024-11-15T23:15:52,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741984_1160 (size=12304) 2024-11-15T23:15:52,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:52,082 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b4692d79f44b415eaed7497479d7bae9_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b4692d79f44b415eaed7497479d7bae9_b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:52,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/26e42a2608884fc4b3fa39cecfb35453, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:52,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/26e42a2608884fc4b3fa39cecfb35453 is 175, key is test_row_0/A:col10/1731712550287/Put/seqid=0 2024-11-15T23:15:52,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741985_1161 (size=31105) 2024-11-15T23:15:52,106 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/26e42a2608884fc4b3fa39cecfb35453 2024-11-15T23:15:52,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/7e056ce2f391408ea79f5010aad985cd is 50, key is test_row_0/B:col10/1731712550287/Put/seqid=0 2024-11-15T23:15:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:52,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:52,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741986_1162 (size=12151) 2024-11-15T23:15:52,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712612228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:52,335 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#A#compaction#132 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:52,336 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/ca2400dae3284f26abf2ceb6344e0569 is 175, key is test_row_0/A:col10/1731712546788/Put/seqid=0 2024-11-15T23:15:52,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741987_1163 (size=31520) 2024-11-15T23:15:52,354 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/ca2400dae3284f26abf2ceb6344e0569 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ca2400dae3284f26abf2ceb6344e0569 2024-11-15T23:15:52,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712612345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:52,364 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in b3f0f65ece63912349255b0111be7183/A of b3f0f65ece63912349255b0111be7183 into ca2400dae3284f26abf2ceb6344e0569(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:52,364 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:52,364 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/A, priority=10, startTime=1731712551869; duration=0sec 2024-11-15T23:15:52,364 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:52,364 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:A 2024-11-15T23:15:52,422 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/19eceed266f14fa7b6492665f8eb4f04 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/19eceed266f14fa7b6492665f8eb4f04 2024-11-15T23:15:52,433 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in b3f0f65ece63912349255b0111be7183/C of b3f0f65ece63912349255b0111be7183 into 19eceed266f14fa7b6492665f8eb4f04(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:52,433 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:52,433 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/C, priority=10, startTime=1731712551869; duration=0sec 2024-11-15T23:15:52,433 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:52,433 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:C 2024-11-15T23:15:52,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712612455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:52,463 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_1' in b3f0f65ece63912349255b0111be7183 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:15:52,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:52,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:52,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:52,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712612466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:52,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712612466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:52,471 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in b3f0f65ece63912349255b0111be7183 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:15:52,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:52,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:52,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:52,471 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in b3f0f65ece63912349255b0111be7183 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:15:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:52,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712612565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:52,567 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/7e056ce2f391408ea79f5010aad985cd 2024-11-15T23:15:52,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/0ae47e4e1a164a999301c4e1c8eb0d7e is 50, key is test_row_0/C:col10/1731712550287/Put/seqid=0 2024-11-15T23:15:52,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741988_1164 (size=12151) 2024-11-15T23:15:52,648 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/0ae47e4e1a164a999301c4e1c8eb0d7e 2024-11-15T23:15:52,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/26e42a2608884fc4b3fa39cecfb35453 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/26e42a2608884fc4b3fa39cecfb35453 2024-11-15T23:15:52,667 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/26e42a2608884fc4b3fa39cecfb35453, entries=150, sequenceid=211, filesize=30.4 K 2024-11-15T23:15:52,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/7e056ce2f391408ea79f5010aad985cd as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/7e056ce2f391408ea79f5010aad985cd 2024-11-15T23:15:52,675 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/7e056ce2f391408ea79f5010aad985cd, entries=150, sequenceid=211, filesize=11.9 K 2024-11-15T23:15:52,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/0ae47e4e1a164a999301c4e1c8eb0d7e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/0ae47e4e1a164a999301c4e1c8eb0d7e 2024-11-15T23:15:52,684 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/0ae47e4e1a164a999301c4e1c8eb0d7e, entries=150, sequenceid=211, filesize=11.9 K 2024-11-15T23:15:52,686 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~365.80 KB/374576, currentSize=53.67 KB/54960 for b3f0f65ece63912349255b0111be7183 in 669ms, sequenceid=211, compaction requested=false 2024-11-15T23:15:52,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:52,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:52,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-15T23:15:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-15T23:15:52,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-15T23:15:52,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9000 sec 2024-11-15T23:15:52,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.9070 sec 2024-11-15T23:15:52,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:52,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T23:15:52,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:15:52,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:52,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:15:52,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:52,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:15:52,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:52,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155962361fabf84e63a7820da4153e0110_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712552226/Put/seqid=0 2024-11-15T23:15:52,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741989_1165 (size=14794) 2024-11-15T23:15:52,917 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:52,923 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155962361fabf84e63a7820da4153e0110_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155962361fabf84e63a7820da4153e0110_b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:52,928 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/0774813748d748018bfe15de9a56d0e6, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:52,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/0774813748d748018bfe15de9a56d0e6 is 175, key is test_row_0/A:col10/1731712552226/Put/seqid=0 2024-11-15T23:15:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T23:15:52,937 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:52,938 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:15:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-15T23:15:52,942 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:15:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T23:15:52,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741990_1166 (size=39749) 2024-11-15T23:15:52,943 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:15:52,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:15:52,949 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/0774813748d748018bfe15de9a56d0e6 2024-11-15T23:15:52,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/348eb998741142f990ff869fdec31316 is 50, key is test_row_0/B:col10/1731712552226/Put/seqid=0 2024-11-15T23:15:52,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741991_1167 (size=12151) 2024-11-15T23:15:52,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/348eb998741142f990ff869fdec31316 2024-11-15T23:15:53,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/82cfed7be66743ca8f98ab45c3b0af4b is 50, key is test_row_0/C:col10/1731712552226/Put/seqid=0 2024-11-15T23:15:53,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712613020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:53,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T23:15:53,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741992_1168 (size=12151) 2024-11-15T23:15:53,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/82cfed7be66743ca8f98ab45c3b0af4b 2024-11-15T23:15:53,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/0774813748d748018bfe15de9a56d0e6 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0774813748d748018bfe15de9a56d0e6 2024-11-15T23:15:53,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0774813748d748018bfe15de9a56d0e6, entries=200, sequenceid=225, filesize=38.8 K 2024-11-15T23:15:53,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/348eb998741142f990ff869fdec31316 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/348eb998741142f990ff869fdec31316 2024-11-15T23:15:53,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/348eb998741142f990ff869fdec31316, entries=150, sequenceid=225, filesize=11.9 K 2024-11-15T23:15:53,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/82cfed7be66743ca8f98ab45c3b0af4b as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/82cfed7be66743ca8f98ab45c3b0af4b 2024-11-15T23:15:53,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/82cfed7be66743ca8f98ab45c3b0af4b, entries=150, sequenceid=225, filesize=11.9 K 2024-11-15T23:15:53,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b3f0f65ece63912349255b0111be7183 in 211ms, sequenceid=225, compaction requested=true 2024-11-15T23:15:53,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:53,093 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:53,094 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102374 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:53,094 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/A is initiating minor compaction (all files) 2024-11-15T23:15:53,094 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/A in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:53,094 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ca2400dae3284f26abf2ceb6344e0569, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/26e42a2608884fc4b3fa39cecfb35453, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0774813748d748018bfe15de9a56d0e6] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=100.0 K 2024-11-15T23:15:53,094 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:53,094 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. files: [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ca2400dae3284f26abf2ceb6344e0569, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/26e42a2608884fc4b3fa39cecfb35453, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0774813748d748018bfe15de9a56d0e6] 2024-11-15T23:15:53,095 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca2400dae3284f26abf2ceb6344e0569, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731712546788 2024-11-15T23:15:53,095 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26e42a2608884fc4b3fa39cecfb35453, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731712550280 2024-11-15T23:15:53,096 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0774813748d748018bfe15de9a56d0e6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731712552208 2024-11-15T23:15:53,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:15:53,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:53,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:15:53,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:53,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:15:53,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T23:15:53,097 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:53,099 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:53,099 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/B is initiating minor compaction (all files) 2024-11-15T23:15:53,099 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/B in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:53,099 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/2fcf0bf87b824156be0bbca63f049600, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/7e056ce2f391408ea79f5010aad985cd, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/348eb998741142f990ff869fdec31316] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=35.9 K 2024-11-15T23:15:53,100 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fcf0bf87b824156be0bbca63f049600, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731712546788 2024-11-15T23:15:53,101 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e056ce2f391408ea79f5010aad985cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731712550280 2024-11-15T23:15:53,102 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 348eb998741142f990ff869fdec31316, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731712552220 2024-11-15T23:15:53,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T23:15:53,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:53,108 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T23:15:53,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:15:53,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:53,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:15:53,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:53,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:15:53,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:53,113 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:53,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159c57a3b2cd29432a9c768b392ef15041_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712552994/Put/seqid=0 2024-11-15T23:15:53,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:53,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:53,139 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115cdb80a7811bc4257b50d782c7e0b6e0c_b3f0f65ece63912349255b0111be7183 store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:53,142 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115cdb80a7811bc4257b50d782c7e0b6e0c_b3f0f65ece63912349255b0111be7183, store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:53,142 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cdb80a7811bc4257b50d782c7e0b6e0c_b3f0f65ece63912349255b0111be7183 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:53,145 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#B#compaction#143 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:53,146 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/1e4aaa9d946441aab04e30d10a9a5d75 is 50, key is test_row_0/B:col10/1731712552226/Put/seqid=0 2024-11-15T23:15:53,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741993_1169 (size=12304) 2024-11-15T23:15:53,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:53,190 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159c57a3b2cd29432a9c768b392ef15041_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159c57a3b2cd29432a9c768b392ef15041_b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:53,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/0924bbaa8e5e48d3b48cff7354e6f9ce, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:53,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/0924bbaa8e5e48d3b48cff7354e6f9ce is 175, key is test_row_0/A:col10/1731712552994/Put/seqid=0 2024-11-15T23:15:53,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741994_1170 (size=4469) 2024-11-15T23:15:53,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:53,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712613221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:53,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741995_1171 (size=12561) 2024-11-15T23:15:53,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T23:15:53,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741996_1172 (size=31105) 2024-11-15T23:15:53,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:53,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712613335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:53,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:53,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:53,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:53,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712613545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:53,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:53,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T23:15:53,614 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#A#compaction#141 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:53,615 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/4a2e80ea953f4be9a3268555a2c4dd23 is 175, key is test_row_0/A:col10/1731712552226/Put/seqid=0 2024-11-15T23:15:53,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741997_1173 (size=31515) 2024-11-15T23:15:53,647 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/1e4aaa9d946441aab04e30d10a9a5d75 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/1e4aaa9d946441aab04e30d10a9a5d75 2024-11-15T23:15:53,657 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3f0f65ece63912349255b0111be7183/B of b3f0f65ece63912349255b0111be7183 into 1e4aaa9d946441aab04e30d10a9a5d75(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:53,657 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:53,658 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/B, priority=13, startTime=1731712553096; duration=0sec 2024-11-15T23:15:53,658 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:53,658 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:B 2024-11-15T23:15:53,658 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:53,660 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:53,660 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/C is initiating minor compaction (all files) 2024-11-15T23:15:53,660 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/C in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:53,660 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/19eceed266f14fa7b6492665f8eb4f04, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/0ae47e4e1a164a999301c4e1c8eb0d7e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/82cfed7be66743ca8f98ab45c3b0af4b] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=35.9 K 2024-11-15T23:15:53,660 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 19eceed266f14fa7b6492665f8eb4f04, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731712546788 2024-11-15T23:15:53,661 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ae47e4e1a164a999301c4e1c8eb0d7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731712550280 2024-11-15T23:15:53,661 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 82cfed7be66743ca8f98ab45c3b0af4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731712552220 2024-11-15T23:15:53,673 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#C#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:53,674 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/e9311061c7e640769fd3a53b58225674 is 50, key is test_row_0/C:col10/1731712552226/Put/seqid=0 2024-11-15T23:15:53,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741998_1174 (size=12561) 2024-11-15T23:15:53,693 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/0924bbaa8e5e48d3b48cff7354e6f9ce 2024-11-15T23:15:53,694 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/e9311061c7e640769fd3a53b58225674 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/e9311061c7e640769fd3a53b58225674 2024-11-15T23:15:53,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/49f0341709ae49e4bf0b65e90b833456 is 50, key is test_row_0/B:col10/1731712552994/Put/seqid=0 2024-11-15T23:15:53,706 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3f0f65ece63912349255b0111be7183/C of b3f0f65ece63912349255b0111be7183 into e9311061c7e640769fd3a53b58225674(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:53,706 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:53,706 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/C, priority=13, startTime=1731712553096; duration=0sec 2024-11-15T23:15:53,706 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:53,706 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:C 2024-11-15T23:15:53,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741999_1175 (size=12151) 2024-11-15T23:15:53,715 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/49f0341709ae49e4bf0b65e90b833456 2024-11-15T23:15:53,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/a3a1c567998f4634b1595e0b6e882fda is 50, key is test_row_0/C:col10/1731712552994/Put/seqid=0 2024-11-15T23:15:53,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742000_1176 (size=12151) 2024-11-15T23:15:53,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712613855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:54,038 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/4a2e80ea953f4be9a3268555a2c4dd23 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/4a2e80ea953f4be9a3268555a2c4dd23 2024-11-15T23:15:54,047 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3f0f65ece63912349255b0111be7183/A of b3f0f65ece63912349255b0111be7183 into 4a2e80ea953f4be9a3268555a2c4dd23(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:54,047 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:54,047 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/A, priority=13, startTime=1731712553093; duration=0sec 2024-11-15T23:15:54,047 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:54,047 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:A 2024-11-15T23:15:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T23:15:54,141 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/a3a1c567998f4634b1595e0b6e882fda 2024-11-15T23:15:54,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/0924bbaa8e5e48d3b48cff7354e6f9ce as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0924bbaa8e5e48d3b48cff7354e6f9ce 2024-11-15T23:15:54,155 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0924bbaa8e5e48d3b48cff7354e6f9ce, entries=150, sequenceid=245, filesize=30.4 K 2024-11-15T23:15:54,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/49f0341709ae49e4bf0b65e90b833456 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/49f0341709ae49e4bf0b65e90b833456 2024-11-15T23:15:54,167 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/49f0341709ae49e4bf0b65e90b833456, entries=150, sequenceid=245, filesize=11.9 K 2024-11-15T23:15:54,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/a3a1c567998f4634b1595e0b6e882fda as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a3a1c567998f4634b1595e0b6e882fda 2024-11-15T23:15:54,176 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a3a1c567998f4634b1595e0b6e882fda, entries=150, sequenceid=245, filesize=11.9 K 2024-11-15T23:15:54,178 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b3f0f65ece63912349255b0111be7183 in 1070ms, sequenceid=245, compaction requested=false 2024-11-15T23:15:54,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:54,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:54,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-15T23:15:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-15T23:15:54,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-15T23:15:54,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2370 sec 2024-11-15T23:15:54,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.2470 sec 2024-11-15T23:15:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:54,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T23:15:54,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:15:54,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:54,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:15:54,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:54,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:15:54,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:54,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111512460595bf0848a582acd54c6a2736bc_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712554369/Put/seqid=0 2024-11-15T23:15:54,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742001_1177 (size=14844) 2024-11-15T23:15:54,400 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:54,405 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111512460595bf0848a582acd54c6a2736bc_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111512460595bf0848a582acd54c6a2736bc_b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:54,411 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/f47cb2b1252c46fd9d46319556631e18, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:54,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/f47cb2b1252c46fd9d46319556631e18 is 175, key is test_row_0/A:col10/1731712554369/Put/seqid=0 2024-11-15T23:15:54,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742002_1178 (size=39799) 2024-11-15T23:15:54,425 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=261, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/f47cb2b1252c46fd9d46319556631e18 2024-11-15T23:15:54,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/d4a2c76d58424049b150411bde65e76c is 50, key is test_row_0/B:col10/1731712554369/Put/seqid=0 2024-11-15T23:15:54,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742003_1179 (size=12201) 2024-11-15T23:15:54,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712614475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:54,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/d4a2c76d58424049b150411bde65e76c 2024-11-15T23:15:54,483 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_1' in b3f0f65ece63912349255b0111be7183 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:15:54,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:54,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:54,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712614478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:54,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:54,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:54,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:54,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:54,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712614486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:54,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712614486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:54,491 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in b3f0f65ece63912349255b0111be7183 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:15:54,491 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in b3f0f65ece63912349255b0111be7183 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:15:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:54,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/a1c87af39835468fb6c2689f6f469eaf is 50, key is test_row_0/C:col10/1731712554369/Put/seqid=0 2024-11-15T23:15:54,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742004_1180 (size=12201) 2024-11-15T23:15:54,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/a1c87af39835468fb6c2689f6f469eaf 2024-11-15T23:15:54,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/f47cb2b1252c46fd9d46319556631e18 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/f47cb2b1252c46fd9d46319556631e18 2024-11-15T23:15:54,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/f47cb2b1252c46fd9d46319556631e18, entries=200, sequenceid=261, filesize=38.9 K 2024-11-15T23:15:54,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/d4a2c76d58424049b150411bde65e76c as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/d4a2c76d58424049b150411bde65e76c 2024-11-15T23:15:54,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/d4a2c76d58424049b150411bde65e76c, entries=150, sequenceid=261, filesize=11.9 K 2024-11-15T23:15:54,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/a1c87af39835468fb6c2689f6f469eaf as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a1c87af39835468fb6c2689f6f469eaf 2024-11-15T23:15:54,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a1c87af39835468fb6c2689f6f469eaf, entries=150, sequenceid=261, filesize=11.9 K 2024-11-15T23:15:54,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b3f0f65ece63912349255b0111be7183 in 191ms, sequenceid=261, compaction requested=true 2024-11-15T23:15:54,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:54,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:15:54,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:54,562 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:54,562 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:54,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:15:54,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:54,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:15:54,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:54,563 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102419 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:54,563 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36913 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:54,563 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/A is initiating minor compaction (all files) 2024-11-15T23:15:54,563 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/B is initiating minor compaction (all files) 2024-11-15T23:15:54,563 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/B in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:54,564 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/1e4aaa9d946441aab04e30d10a9a5d75, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/49f0341709ae49e4bf0b65e90b833456, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/d4a2c76d58424049b150411bde65e76c] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=36.0 K 2024-11-15T23:15:54,564 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/A in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:54,564 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/4a2e80ea953f4be9a3268555a2c4dd23, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0924bbaa8e5e48d3b48cff7354e6f9ce, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/f47cb2b1252c46fd9d46319556631e18] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=100.0 K 2024-11-15T23:15:54,564 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:54,564 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. files: [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/4a2e80ea953f4be9a3268555a2c4dd23, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0924bbaa8e5e48d3b48cff7354e6f9ce, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/f47cb2b1252c46fd9d46319556631e18] 2024-11-15T23:15:54,566 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e4aaa9d946441aab04e30d10a9a5d75, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731712552220 2024-11-15T23:15:54,567 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a2e80ea953f4be9a3268555a2c4dd23, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731712552220 2024-11-15T23:15:54,568 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0924bbaa8e5e48d3b48cff7354e6f9ce, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731712552994 2024-11-15T23:15:54,570 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 49f0341709ae49e4bf0b65e90b833456, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731712552994 2024-11-15T23:15:54,570 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting f47cb2b1252c46fd9d46319556631e18, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731712553177 2024-11-15T23:15:54,571 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting d4a2c76d58424049b150411bde65e76c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731712553177 2024-11-15T23:15:54,595 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:54,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T23:15:54,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:15:54,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:54,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:15:54,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:54,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:15:54,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:54,613 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#B#compaction#151 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:54,613 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/d2755d8eeb874f49b20a814391c75758 is 50, key is test_row_0/B:col10/1731712554369/Put/seqid=0 2024-11-15T23:15:54,621 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111582f872e1aff14ce582bc91170431edfd_b3f0f65ece63912349255b0111be7183 store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:54,624 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111582f872e1aff14ce582bc91170431edfd_b3f0f65ece63912349255b0111be7183, store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:54,624 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111582f872e1aff14ce582bc91170431edfd_b3f0f65ece63912349255b0111be7183 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:54,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153ecee291601c4887bed642e15ebce1b1_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712554458/Put/seqid=0 2024-11-15T23:15:54,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712614666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:54,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742005_1181 (size=12713) 2024-11-15T23:15:54,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742006_1182 (size=4469) 2024-11-15T23:15:54,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742007_1183 (size=14994) 2024-11-15T23:15:54,769 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:54,774 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153ecee291601c4887bed642e15ebce1b1_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153ecee291601c4887bed642e15ebce1b1_b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:54,775 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/95231691bcb648a6869c888b4eabb08f, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:54,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/95231691bcb648a6869c888b4eabb08f is 175, key is test_row_0/A:col10/1731712554458/Put/seqid=0 2024-11-15T23:15:54,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712614775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:54,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742008_1184 (size=39949) 2024-11-15T23:15:54,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712614985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:54,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:54,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:54,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T23:15:55,095 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:55,096 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:15:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-15T23:15:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T23:15:55,097 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:15:55,098 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:15:55,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:15:55,112 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/d2755d8eeb874f49b20a814391c75758 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/d2755d8eeb874f49b20a814391c75758 2024-11-15T23:15:55,117 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3f0f65ece63912349255b0111be7183/B of b3f0f65ece63912349255b0111be7183 into d2755d8eeb874f49b20a814391c75758(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:55,117 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:55,117 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/B, priority=13, startTime=1731712554562; duration=0sec 2024-11-15T23:15:55,117 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:55,117 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:B 2024-11-15T23:15:55,117 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:55,118 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36913 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:55,119 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/C is initiating minor compaction (all files) 2024-11-15T23:15:55,119 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/C in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:55,119 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/e9311061c7e640769fd3a53b58225674, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a3a1c567998f4634b1595e0b6e882fda, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a1c87af39835468fb6c2689f6f469eaf] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=36.0 K 2024-11-15T23:15:55,119 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting e9311061c7e640769fd3a53b58225674, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731712552220 2024-11-15T23:15:55,120 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting a3a1c567998f4634b1595e0b6e882fda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731712552994 2024-11-15T23:15:55,120 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting a1c87af39835468fb6c2689f6f469eaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731712553177 2024-11-15T23:15:55,127 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#C#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:55,128 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/7a56cd6062b74fb2a69bc721c5ec82f3 is 50, key is test_row_0/C:col10/1731712554369/Put/seqid=0 2024-11-15T23:15:55,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742009_1185 (size=12713) 2024-11-15T23:15:55,143 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#A#compaction#150 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:55,144 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/aad8b02cba5c43a086b9fe79862876a2 is 175, key is test_row_0/A:col10/1731712554369/Put/seqid=0 2024-11-15T23:15:55,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742010_1186 (size=31667) 2024-11-15T23:15:55,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T23:15:55,207 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/95231691bcb648a6869c888b4eabb08f 2024-11-15T23:15:55,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/d0fdaae61dfe4a98ab8084adf6abbffc is 50, key is test_row_0/B:col10/1731712554458/Put/seqid=0 2024-11-15T23:15:55,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742011_1187 (size=12301) 2024-11-15T23:15:55,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/d0fdaae61dfe4a98ab8084adf6abbffc 2024-11-15T23:15:55,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/ecd76d33651f4b25b5deac2f30fe0959 is 50, key is test_row_0/C:col10/1731712554458/Put/seqid=0 2024-11-15T23:15:55,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T23:15:55,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:55,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:55,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:55,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:55,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:55,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742012_1188 (size=12301) 2024-11-15T23:15:55,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/ecd76d33651f4b25b5deac2f30fe0959 2024-11-15T23:15:55,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/95231691bcb648a6869c888b4eabb08f as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/95231691bcb648a6869c888b4eabb08f 2024-11-15T23:15:55,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/95231691bcb648a6869c888b4eabb08f, entries=200, sequenceid=281, filesize=39.0 K 2024-11-15T23:15:55,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/d0fdaae61dfe4a98ab8084adf6abbffc as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/d0fdaae61dfe4a98ab8084adf6abbffc 2024-11-15T23:15:55,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/d0fdaae61dfe4a98ab8084adf6abbffc, entries=150, sequenceid=281, filesize=12.0 K 2024-11-15T23:15:55,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/ecd76d33651f4b25b5deac2f30fe0959 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/ecd76d33651f4b25b5deac2f30fe0959 2024-11-15T23:15:55,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/ecd76d33651f4b25b5deac2f30fe0959, entries=150, sequenceid=281, filesize=12.0 K 2024-11-15T23:15:55,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b3f0f65ece63912349255b0111be7183 in 673ms, sequenceid=281, compaction requested=false 2024-11-15T23:15:55,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:55,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T23:15:55,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:15:55,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:55,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:15:55,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:55,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:15:55,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:55,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fed89b4ed87342f49b8c26b099fb65f3_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712554653/Put/seqid=0 2024-11-15T23:15:55,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742013_1189 (size=14994) 2024-11-15T23:15:55,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:55,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712615354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:55,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:55,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:55,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:55,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T23:15:55,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:55,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:55,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:55,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:55,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T23:15:55,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712615465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:55,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:55,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:55,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:55,541 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/7a56cd6062b74fb2a69bc721c5ec82f3 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/7a56cd6062b74fb2a69bc721c5ec82f3 2024-11-15T23:15:55,551 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3f0f65ece63912349255b0111be7183/C of b3f0f65ece63912349255b0111be7183 into 7a56cd6062b74fb2a69bc721c5ec82f3(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:55,551 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:55,551 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/C, priority=13, startTime=1731712554562; duration=0sec 2024-11-15T23:15:55,553 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:55,553 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:C 2024-11-15T23:15:55,557 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/aad8b02cba5c43a086b9fe79862876a2 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/aad8b02cba5c43a086b9fe79862876a2 2024-11-15T23:15:55,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T23:15:55,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:55,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:55,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:55,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:55,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:55,564 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3f0f65ece63912349255b0111be7183/A of b3f0f65ece63912349255b0111be7183 into aad8b02cba5c43a086b9fe79862876a2(size=30.9 K), total size for store is 69.9 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-15T23:15:55,564 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:55,564 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/A, priority=13, startTime=1731712554562; duration=1sec 2024-11-15T23:15:55,564 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:55,564 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:A 2024-11-15T23:15:55,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712615676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:55,709 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:55,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T23:15:55,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:55,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:55,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:55,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:55,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:55,717 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fed89b4ed87342f49b8c26b099fb65f3_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fed89b4ed87342f49b8c26b099fb65f3_b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:55,719 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/862ba4ae2ea54277a2279be291271fb0, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:55,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/862ba4ae2ea54277a2279be291271fb0 is 175, key is test_row_0/A:col10/1731712554653/Put/seqid=0 2024-11-15T23:15:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T23:15:55,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742014_1190 (size=39949) 2024-11-15T23:15:55,732 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/862ba4ae2ea54277a2279be291271fb0 2024-11-15T23:15:55,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/62f74adf351c47a186690beee575d90e is 50, key is test_row_0/B:col10/1731712554653/Put/seqid=0 2024-11-15T23:15:55,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742015_1191 (size=12301) 2024-11-15T23:15:55,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/62f74adf351c47a186690beee575d90e 2024-11-15T23:15:55,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/a1ae3f616ec5439b96e5dc3d64783ece is 50, key is test_row_0/C:col10/1731712554653/Put/seqid=0 2024-11-15T23:15:55,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742016_1192 (size=12301) 2024-11-15T23:15:55,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/a1ae3f616ec5439b96e5dc3d64783ece 2024-11-15T23:15:55,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/862ba4ae2ea54277a2279be291271fb0 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/862ba4ae2ea54277a2279be291271fb0 2024-11-15T23:15:55,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/862ba4ae2ea54277a2279be291271fb0, entries=200, sequenceid=295, filesize=39.0 K 2024-11-15T23:15:55,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/62f74adf351c47a186690beee575d90e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/62f74adf351c47a186690beee575d90e 2024-11-15T23:15:55,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/62f74adf351c47a186690beee575d90e, entries=150, sequenceid=295, filesize=12.0 K 2024-11-15T23:15:55,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/a1ae3f616ec5439b96e5dc3d64783ece as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a1ae3f616ec5439b96e5dc3d64783ece 2024-11-15T23:15:55,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a1ae3f616ec5439b96e5dc3d64783ece, entries=150, sequenceid=295, filesize=12.0 K 2024-11-15T23:15:55,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b3f0f65ece63912349255b0111be7183 in 560ms, sequenceid=295, compaction requested=true 2024-11-15T23:15:55,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:55,858 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:55,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:15:55,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:55,861 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:55,862 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111565 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:55,862 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/A is initiating minor compaction (all files) 2024-11-15T23:15:55,862 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/A in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:55,862 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/aad8b02cba5c43a086b9fe79862876a2, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/95231691bcb648a6869c888b4eabb08f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/862ba4ae2ea54277a2279be291271fb0] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=109.0 K 2024-11-15T23:15:55,862 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:55,862 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. files: [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/aad8b02cba5c43a086b9fe79862876a2, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/95231691bcb648a6869c888b4eabb08f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/862ba4ae2ea54277a2279be291271fb0] 2024-11-15T23:15:55,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T23:15:55,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:55,864 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T23:15:55,864 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting aad8b02cba5c43a086b9fe79862876a2, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731712553177 2024-11-15T23:15:55,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:15:55,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:55,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:15:55,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:15:55,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:55,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:15:55,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:55,865 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37315 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:55,865 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/B is initiating minor compaction (all files) 2024-11-15T23:15:55,866 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/B in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:55,866 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/d2755d8eeb874f49b20a814391c75758, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/d0fdaae61dfe4a98ab8084adf6abbffc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/62f74adf351c47a186690beee575d90e] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=36.4 K 2024-11-15T23:15:55,866 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95231691bcb648a6869c888b4eabb08f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731712554443 2024-11-15T23:15:55,866 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting d2755d8eeb874f49b20a814391c75758, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731712553177 2024-11-15T23:15:55,867 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 862ba4ae2ea54277a2279be291271fb0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731712554631 2024-11-15T23:15:55,867 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting d0fdaae61dfe4a98ab8084adf6abbffc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731712554443 2024-11-15T23:15:55,868 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 62f74adf351c47a186690beee575d90e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731712554631 2024-11-15T23:15:55,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:55,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:15:55,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:55,886 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:55,907 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#B#compaction#160 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:55,907 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/97b415be228e48fa83afe2f8d5b217c4 is 50, key is test_row_0/B:col10/1731712554653/Put/seqid=0 2024-11-15T23:15:55,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153f6cae7cb648440cb25a45785ef16dda_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712555352/Put/seqid=0 2024-11-15T23:15:55,921 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115d6c40fe79924420aafad15f30ab52563_b3f0f65ece63912349255b0111be7183 store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:55,923 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115d6c40fe79924420aafad15f30ab52563_b3f0f65ece63912349255b0111be7183, store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:55,923 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d6c40fe79924420aafad15f30ab52563_b3f0f65ece63912349255b0111be7183 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:55,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742017_1193 (size=12915) 2024-11-15T23:15:55,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742018_1194 (size=12454) 2024-11-15T23:15:55,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:55,958 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153f6cae7cb648440cb25a45785ef16dda_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153f6cae7cb648440cb25a45785ef16dda_b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:55,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/fcf428ac917b45748aa3830b8179b29f, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:55,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/fcf428ac917b45748aa3830b8179b29f is 175, key is test_row_0/A:col10/1731712555352/Put/seqid=0 2024-11-15T23:15:55,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742019_1195 (size=4469) 2024-11-15T23:15:55,969 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#A#compaction#159 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:55,970 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/11ea646fa195481d8a55215ba64962d5 is 175, key is test_row_0/A:col10/1731712554653/Put/seqid=0 2024-11-15T23:15:55,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742021_1197 (size=31869) 2024-11-15T23:15:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:55,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:55,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742020_1196 (size=31255) 2024-11-15T23:15:56,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712616041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:56,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:56,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:56,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:56,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712616155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T23:15:56,356 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/97b415be228e48fa83afe2f8d5b217c4 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/97b415be228e48fa83afe2f8d5b217c4 2024-11-15T23:15:56,364 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3f0f65ece63912349255b0111be7183/B of b3f0f65ece63912349255b0111be7183 into 97b415be228e48fa83afe2f8d5b217c4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:56,364 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:56,364 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/B, priority=13, startTime=1731712555861; duration=0sec 2024-11-15T23:15:56,364 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:56,364 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:B 2024-11-15T23:15:56,364 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:56,365 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37315 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:56,365 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/C is initiating minor compaction (all files) 2024-11-15T23:15:56,366 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/C in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:56,366 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/7a56cd6062b74fb2a69bc721c5ec82f3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/ecd76d33651f4b25b5deac2f30fe0959, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a1ae3f616ec5439b96e5dc3d64783ece] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=36.4 K 2024-11-15T23:15:56,366 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a56cd6062b74fb2a69bc721c5ec82f3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731712553177 2024-11-15T23:15:56,366 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting ecd76d33651f4b25b5deac2f30fe0959, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731712554443 2024-11-15T23:15:56,367 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting a1ae3f616ec5439b96e5dc3d64783ece, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731712554631 2024-11-15T23:15:56,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:56,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712616365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:56,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:56,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:56,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:56,375 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#C#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:56,375 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/a6986d71f3e9417395cad866ef9c700c is 50, key is test_row_0/C:col10/1731712554653/Put/seqid=0 2024-11-15T23:15:56,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742022_1198 (size=12915) 2024-11-15T23:15:56,391 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/11ea646fa195481d8a55215ba64962d5 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/11ea646fa195481d8a55215ba64962d5 2024-11-15T23:15:56,397 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3f0f65ece63912349255b0111be7183/A of b3f0f65ece63912349255b0111be7183 into 11ea646fa195481d8a55215ba64962d5(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:56,397 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:56,397 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/A, priority=13, startTime=1731712555858; duration=0sec 2024-11-15T23:15:56,397 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:56,397 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:A 2024-11-15T23:15:56,400 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/fcf428ac917b45748aa3830b8179b29f 2024-11-15T23:15:56,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/4ebc2e7bed354a6a8917a6d9a122bebb is 50, key is test_row_0/B:col10/1731712555352/Put/seqid=0 2024-11-15T23:15:56,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742023_1199 (size=12301) 2024-11-15T23:15:56,432 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/4ebc2e7bed354a6a8917a6d9a122bebb 2024-11-15T23:15:56,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/b0ee33f751f840869c96832dae852167 is 50, key is test_row_0/C:col10/1731712555352/Put/seqid=0 2024-11-15T23:15:56,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742024_1200 (size=12301) 2024-11-15T23:15:56,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712616675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:56,784 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/a6986d71f3e9417395cad866ef9c700c as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a6986d71f3e9417395cad866ef9c700c 2024-11-15T23:15:56,788 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3f0f65ece63912349255b0111be7183/C of b3f0f65ece63912349255b0111be7183 into a6986d71f3e9417395cad866ef9c700c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:56,788 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:56,788 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/C, priority=13, startTime=1731712555869; duration=0sec 2024-11-15T23:15:56,788 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:56,788 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:C 2024-11-15T23:15:56,851 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/b0ee33f751f840869c96832dae852167 2024-11-15T23:15:56,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/fcf428ac917b45748aa3830b8179b29f as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/fcf428ac917b45748aa3830b8179b29f 2024-11-15T23:15:56,861 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/fcf428ac917b45748aa3830b8179b29f, entries=150, sequenceid=316, filesize=30.5 K 2024-11-15T23:15:56,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/4ebc2e7bed354a6a8917a6d9a122bebb as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/4ebc2e7bed354a6a8917a6d9a122bebb 2024-11-15T23:15:56,867 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/4ebc2e7bed354a6a8917a6d9a122bebb, entries=150, sequenceid=316, filesize=12.0 K 2024-11-15T23:15:56,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/b0ee33f751f840869c96832dae852167 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/b0ee33f751f840869c96832dae852167 2024-11-15T23:15:56,873 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/b0ee33f751f840869c96832dae852167, entries=150, sequenceid=316, filesize=12.0 K 2024-11-15T23:15:56,874 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b3f0f65ece63912349255b0111be7183 in 1010ms, sequenceid=316, compaction requested=false 2024-11-15T23:15:56,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:56,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:56,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-15T23:15:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-15T23:15:56,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-15T23:15:56,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7770 sec 2024-11-15T23:15:56,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.7810 sec 2024-11-15T23:15:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:57,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T23:15:57,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:15:57,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:57,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:15:57,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:57,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:15:57,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:57,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ed9edb89f48340f5bd3f7a23c2dc1376_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712557187/Put/seqid=0 2024-11-15T23:15:57,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742025_1201 (size=14994) 2024-11-15T23:15:57,211 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:57,222 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ed9edb89f48340f5bd3f7a23c2dc1376_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ed9edb89f48340f5bd3f7a23c2dc1376_b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:57,223 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/c6239ecf05b94a059e4d4f69d5031d25, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:57,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/c6239ecf05b94a059e4d4f69d5031d25 is 175, key is test_row_0/A:col10/1731712557187/Put/seqid=0 2024-11-15T23:15:57,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742026_1202 (size=39949) 2024-11-15T23:15:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T23:15:57,246 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:15:57,246 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:15:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-15T23:15:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T23:15:57,247 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:15:57,248 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:15:57,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:15:57,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712617259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T23:15:57,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712617365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:57,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:57,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:57,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:57,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T23:15:57,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:57,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:57,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:57,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:57,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:57,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T23:15:57,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:57,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:57,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:57,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:57,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T23:15:57,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712617575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:57,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:57,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:57,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:57,636 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/c6239ecf05b94a059e4d4f69d5031d25 2024-11-15T23:15:57,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/9a9d1ffb39be4c968e2da3ee6b6345d7 is 50, key is test_row_0/B:col10/1731712557187/Put/seqid=0 2024-11-15T23:15:57,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742027_1203 (size=12301) 2024-11-15T23:15:57,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T23:15:57,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:57,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:57,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:57,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:57,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:57,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:57,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T23:15:57,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:57,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:57,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:57,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:57,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:57,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T23:15:57,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712617885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:57,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:57,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:57,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:58,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T23:15:58,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:58,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:58,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:58,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/9a9d1ffb39be4c968e2da3ee6b6345d7 2024-11-15T23:15:58,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/d1fe93065a4b4fd792d41400041de3b6 is 50, key is test_row_0/C:col10/1731712557187/Put/seqid=0 2024-11-15T23:15:58,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742028_1204 (size=12301) 2024-11-15T23:15:58,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T23:15:58,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:58,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:58,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:58,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T23:15:58,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:58,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:58,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:58,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T23:15:58,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712618405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:58,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T23:15:58,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:58,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:58,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:58,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/d1fe93065a4b4fd792d41400041de3b6 2024-11-15T23:15:58,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/c6239ecf05b94a059e4d4f69d5031d25 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/c6239ecf05b94a059e4d4f69d5031d25 2024-11-15T23:15:58,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/c6239ecf05b94a059e4d4f69d5031d25, entries=200, sequenceid=333, filesize=39.0 K 2024-11-15T23:15:58,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/9a9d1ffb39be4c968e2da3ee6b6345d7 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/9a9d1ffb39be4c968e2da3ee6b6345d7 2024-11-15T23:15:58,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/9a9d1ffb39be4c968e2da3ee6b6345d7, entries=150, sequenceid=333, filesize=12.0 K 2024-11-15T23:15:58,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/d1fe93065a4b4fd792d41400041de3b6 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/d1fe93065a4b4fd792d41400041de3b6 2024-11-15T23:15:58,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/d1fe93065a4b4fd792d41400041de3b6, entries=150, sequenceid=333, filesize=12.0 K 2024-11-15T23:15:58,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b3f0f65ece63912349255b0111be7183 in 1333ms, sequenceid=333, compaction requested=true 2024-11-15T23:15:58,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:58,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:15:58,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:58,521 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:58,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:15:58,521 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:58,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:58,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:15:58,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:58,522 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:58,522 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:58,522 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/A is initiating minor compaction (all files) 2024-11-15T23:15:58,522 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/B is initiating minor compaction (all files) 2024-11-15T23:15:58,523 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/B in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:58,523 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/A in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:58,523 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/11ea646fa195481d8a55215ba64962d5, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/fcf428ac917b45748aa3830b8179b29f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/c6239ecf05b94a059e4d4f69d5031d25] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=100.7 K 2024-11-15T23:15:58,523 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/97b415be228e48fa83afe2f8d5b217c4, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/4ebc2e7bed354a6a8917a6d9a122bebb, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/9a9d1ffb39be4c968e2da3ee6b6345d7] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=36.6 K 2024-11-15T23:15:58,523 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:58,523 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. files: [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/11ea646fa195481d8a55215ba64962d5, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/fcf428ac917b45748aa3830b8179b29f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/c6239ecf05b94a059e4d4f69d5031d25] 2024-11-15T23:15:58,523 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 97b415be228e48fa83afe2f8d5b217c4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731712554631 2024-11-15T23:15:58,523 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11ea646fa195481d8a55215ba64962d5, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731712554631 2024-11-15T23:15:58,523 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ebc2e7bed354a6a8917a6d9a122bebb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731712555344 2024-11-15T23:15:58,523 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcf428ac917b45748aa3830b8179b29f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731712555344 2024-11-15T23:15:58,524 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a9d1ffb39be4c968e2da3ee6b6345d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731712556015 2024-11-15T23:15:58,524 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6239ecf05b94a059e4d4f69d5031d25, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731712556015 2024-11-15T23:15:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:58,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T23:15:58,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:15:58,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:58,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:15:58,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:58,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:15:58,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:58,531 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:58,533 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#B#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:58,533 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/4bcd44b755f84ac59e19ea9638ef3e71 is 50, key is test_row_0/B:col10/1731712557187/Put/seqid=0 2024-11-15T23:15:58,536 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115ef048948de0f410c8010306bd19ae52f_b3f0f65ece63912349255b0111be7183 store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:58,538 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115ef048948de0f410c8010306bd19ae52f_b3f0f65ece63912349255b0111be7183, store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:58,538 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ef048948de0f410c8010306bd19ae52f_b3f0f65ece63912349255b0111be7183 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:58,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742029_1205 (size=13017) 2024-11-15T23:15:58,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712618544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:58,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712618546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:58,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:58,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:58,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712618548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:58,552 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/4bcd44b755f84ac59e19ea9638ef3e71 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/4bcd44b755f84ac59e19ea9638ef3e71 2024-11-15T23:15:58,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:58,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156e714c64263b4f098d6b1c4302365c03_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712557236/Put/seqid=0 2024-11-15T23:15:58,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742030_1206 (size=4469) 2024-11-15T23:15:58,563 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#A#compaction#169 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:58,563 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/ef58bedac4ce437da383e1c2d90d9349 is 175, key is test_row_0/A:col10/1731712557187/Put/seqid=0 2024-11-15T23:15:58,564 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3f0f65ece63912349255b0111be7183/B of b3f0f65ece63912349255b0111be7183 into 4bcd44b755f84ac59e19ea9638ef3e71(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:58,564 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:58,564 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/B, priority=13, startTime=1731712558521; duration=0sec 2024-11-15T23:15:58,564 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:15:58,564 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:B 2024-11-15T23:15:58,564 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:15:58,565 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:15:58,565 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/C is initiating minor compaction (all files) 2024-11-15T23:15:58,565 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/C in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:58,566 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a6986d71f3e9417395cad866ef9c700c, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/b0ee33f751f840869c96832dae852167, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/d1fe93065a4b4fd792d41400041de3b6] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=36.6 K 2024-11-15T23:15:58,566 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting a6986d71f3e9417395cad866ef9c700c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731712554631 2024-11-15T23:15:58,566 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting b0ee33f751f840869c96832dae852167, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731712555344 2024-11-15T23:15:58,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742031_1207 (size=17534) 2024-11-15T23:15:58,568 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting d1fe93065a4b4fd792d41400041de3b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731712556015 2024-11-15T23:15:58,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742032_1208 (size=31971) 2024-11-15T23:15:58,598 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#C#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:15:58,599 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/b5e72eb9d06e436ca557f73594d19927 is 50, key is test_row_0/C:col10/1731712557187/Put/seqid=0 2024-11-15T23:15:58,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742033_1209 (size=13017) 2024-11-15T23:15:58,626 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/b5e72eb9d06e436ca557f73594d19927 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/b5e72eb9d06e436ca557f73594d19927 2024-11-15T23:15:58,630 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3f0f65ece63912349255b0111be7183/C of b3f0f65ece63912349255b0111be7183 into b5e72eb9d06e436ca557f73594d19927(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:58,630 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:58,631 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/C, priority=13, startTime=1731712558521; duration=0sec 2024-11-15T23:15:58,631 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:58,631 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:C 2024-11-15T23:15:58,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T23:15:58,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:58,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:58,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:58,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712618655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:58,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:58,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:58,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712618655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:58,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:58,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712618655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:58,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:58,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T23:15:58,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:58,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:58,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:58,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:58,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712618865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:58,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712618866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:58,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712618866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:58,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:58,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:58,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:58,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:58,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T23:15:58,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:58,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:58,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:58,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:58,974 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:58,980 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156e714c64263b4f098d6b1c4302365c03_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156e714c64263b4f098d6b1c4302365c03_b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:58,982 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/46b8ba7c2ae7467b95e1e164b8885a0b, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:58,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/46b8ba7c2ae7467b95e1e164b8885a0b is 175, key is test_row_0/A:col10/1731712557236/Put/seqid=0 2024-11-15T23:15:59,010 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/ef58bedac4ce437da383e1c2d90d9349 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ef58bedac4ce437da383e1c2d90d9349 2024-11-15T23:15:59,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742034_1210 (size=48639) 2024-11-15T23:15:59,026 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3f0f65ece63912349255b0111be7183/A of b3f0f65ece63912349255b0111be7183 into ef58bedac4ce437da383e1c2d90d9349(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:15:59,026 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:59,026 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/A, priority=13, startTime=1731712558521; duration=0sec 2024-11-15T23:15:59,027 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:15:59,027 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:A 2024-11-15T23:15:59,027 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/46b8ba7c2ae7467b95e1e164b8885a0b 2024-11-15T23:15:59,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/662831dc07e94f498f5ea121f71e9de9 is 50, key is test_row_0/B:col10/1731712557236/Put/seqid=0 2024-11-15T23:15:59,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742035_1211 (size=12301) 2024-11-15T23:15:59,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/662831dc07e94f498f5ea121f71e9de9 2024-11-15T23:15:59,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/8150d02f6e01488dbcfe684fb278945e is 50, key is test_row_0/C:col10/1731712557236/Put/seqid=0 2024-11-15T23:15:59,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742036_1212 (size=12301) 2024-11-15T23:15:59,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T23:15:59,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:59,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:59,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:59,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:59,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:59,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712619175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:59,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:59,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712619175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:59,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:59,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:59,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712619175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:59,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:59,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T23:15:59,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:59,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:59,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:59,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:59,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T23:15:59,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T23:15:59,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:59,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:59,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:59,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:59,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:15:59,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50748 deadline: 1731712619415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:59,416 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in b3f0f65ece63912349255b0111be7183 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:15:59,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:59,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/8150d02f6e01488dbcfe684fb278945e 2024-11-15T23:15:59,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/46b8ba7c2ae7467b95e1e164b8885a0b as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/46b8ba7c2ae7467b95e1e164b8885a0b 2024-11-15T23:15:59,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/46b8ba7c2ae7467b95e1e164b8885a0b, entries=250, sequenceid=353, filesize=47.5 K 2024-11-15T23:15:59,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/662831dc07e94f498f5ea121f71e9de9 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/662831dc07e94f498f5ea121f71e9de9 2024-11-15T23:15:59,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/662831dc07e94f498f5ea121f71e9de9, entries=150, sequenceid=353, filesize=12.0 K 2024-11-15T23:15:59,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/8150d02f6e01488dbcfe684fb278945e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/8150d02f6e01488dbcfe684fb278945e 2024-11-15T23:15:59,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/8150d02f6e01488dbcfe684fb278945e, entries=150, sequenceid=353, filesize=12.0 K 2024-11-15T23:15:59,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b3f0f65ece63912349255b0111be7183 in 968ms, sequenceid=353, compaction requested=false 2024-11-15T23:15:59,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:15:59,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T23:15:59,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:15:59,556 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T23:15:59,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:15:59,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:59,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:15:59,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:59,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:15:59,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:15:59,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a0ccdc3f2da04210878835673fad6e8b_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712558534/Put/seqid=0 2024-11-15T23:15:59,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742037_1213 (size=12454) 2024-11-15T23:15:59,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. as already flushing 2024-11-15T23:15:59,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:59,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:59,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712619718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:59,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:59,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712619718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:59,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:59,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712619719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:59,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712619825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:59,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712619825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:59,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:15:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712619825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:15:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:15:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:15:59,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:15:59,983 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a0ccdc3f2da04210878835673fad6e8b_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a0ccdc3f2da04210878835673fad6e8b_b3f0f65ece63912349255b0111be7183 2024-11-15T23:15:59,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/f95e1da0ef4b40aa98f7558f7df72822, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:15:59,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/f95e1da0ef4b40aa98f7558f7df72822 is 175, key is test_row_0/A:col10/1731712558534/Put/seqid=0 2024-11-15T23:16:00,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742038_1214 (size=31255) 2024-11-15T23:16:00,002 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/f95e1da0ef4b40aa98f7558f7df72822 2024-11-15T23:16:00,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/32825a92e6a44126b801907c9c224d3e is 50, key is test_row_0/B:col10/1731712558534/Put/seqid=0 2024-11-15T23:16:00,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742039_1215 (size=12301) 2024-11-15T23:16:00,023 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/32825a92e6a44126b801907c9c224d3e 2024-11-15T23:16:00,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/e88cf70a8a8c43b6a14ba71a4d3d9416 is 50, key is test_row_0/C:col10/1731712558534/Put/seqid=0 2024-11-15T23:16:00,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712620035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:00,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:00,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:00,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:00,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712620036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:00,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:00,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:00,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712620036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:00,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:00,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:00,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:00,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:00,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:00,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742040_1216 (size=12301) 2024-11-15T23:16:00,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712620345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:00,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712620345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:00,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712620345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:00,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:00,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:00,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:00,457 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/e88cf70a8a8c43b6a14ba71a4d3d9416 2024-11-15T23:16:00,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/f95e1da0ef4b40aa98f7558f7df72822 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/f95e1da0ef4b40aa98f7558f7df72822 2024-11-15T23:16:00,465 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/f95e1da0ef4b40aa98f7558f7df72822, entries=150, sequenceid=368, filesize=30.5 K 2024-11-15T23:16:00,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/32825a92e6a44126b801907c9c224d3e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/32825a92e6a44126b801907c9c224d3e 2024-11-15T23:16:00,472 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/32825a92e6a44126b801907c9c224d3e, entries=150, sequenceid=368, filesize=12.0 K 2024-11-15T23:16:00,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/e88cf70a8a8c43b6a14ba71a4d3d9416 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/e88cf70a8a8c43b6a14ba71a4d3d9416 2024-11-15T23:16:00,478 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/e88cf70a8a8c43b6a14ba71a4d3d9416, entries=150, sequenceid=368, filesize=12.0 K 2024-11-15T23:16:00,479 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b3f0f65ece63912349255b0111be7183 in 923ms, sequenceid=368, compaction requested=true 2024-11-15T23:16:00,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:16:00,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:16:00,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-15T23:16:00,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-15T23:16:00,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-15T23:16:00,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2340 sec 2024-11-15T23:16:00,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 3.2380 sec 2024-11-15T23:16:00,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-15T23:16:00,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:16:00,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:00,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:16:00,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:00,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:16:00,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:00,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:00,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151266a4924cfa4df7abe4631b4739129c_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712559717/Put/seqid=0 2024-11-15T23:16:00,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50810 deadline: 1731712620868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:00,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:00,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50838 deadline: 1731712620875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50778 deadline: 1731712620875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:00,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:00,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:00,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:00,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742041_1217 (size=14994) 2024-11-15T23:16:00,885 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:00,890 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151266a4924cfa4df7abe4631b4739129c_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151266a4924cfa4df7abe4631b4739129c_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:00,892 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/0be5e6495eb44eb1b5b29de7248f5b55, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:16:00,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/0be5e6495eb44eb1b5b29de7248f5b55 is 175, key is test_row_0/A:col10/1731712559717/Put/seqid=0 2024-11-15T23:16:00,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742042_1218 (size=39949) 2024-11-15T23:16:00,900 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=389, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/0be5e6495eb44eb1b5b29de7248f5b55 2024-11-15T23:16:00,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/4aced2d8282c4e29a3ec7bb3ff410b5f is 50, key is test_row_0/B:col10/1731712559717/Put/seqid=0 2024-11-15T23:16:00,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742043_1219 (size=12301) 2024-11-15T23:16:00,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/4aced2d8282c4e29a3ec7bb3ff410b5f 2024-11-15T23:16:00,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/a0f7856ccb234ced92d6b7f83316fd39 is 50, key is test_row_0/C:col10/1731712559717/Put/seqid=0 2024-11-15T23:16:00,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742044_1220 (size=12301) 2024-11-15T23:16:00,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/a0f7856ccb234ced92d6b7f83316fd39 2024-11-15T23:16:00,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/0be5e6495eb44eb1b5b29de7248f5b55 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0be5e6495eb44eb1b5b29de7248f5b55 2024-11-15T23:16:00,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0be5e6495eb44eb1b5b29de7248f5b55, entries=200, sequenceid=389, filesize=39.0 K 2024-11-15T23:16:00,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/4aced2d8282c4e29a3ec7bb3ff410b5f as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/4aced2d8282c4e29a3ec7bb3ff410b5f 2024-11-15T23:16:00,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/4aced2d8282c4e29a3ec7bb3ff410b5f, entries=150, sequenceid=389, filesize=12.0 K 2024-11-15T23:16:00,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/a0f7856ccb234ced92d6b7f83316fd39 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a0f7856ccb234ced92d6b7f83316fd39 2024-11-15T23:16:00,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a0f7856ccb234ced92d6b7f83316fd39, entries=150, sequenceid=389, filesize=12.0 K 2024-11-15T23:16:00,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for b3f0f65ece63912349255b0111be7183 in 108ms, sequenceid=389, compaction requested=true 2024-11-15T23:16:00,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:16:00,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:16:00,965 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:16:00,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:00,965 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:16:00,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:16:00,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:00,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3f0f65ece63912349255b0111be7183:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:16:00,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:00,967 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151814 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:16:00,967 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:16:00,967 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/A is initiating minor compaction (all files) 2024-11-15T23:16:00,967 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/B is initiating minor compaction (all files) 2024-11-15T23:16:00,967 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/B in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:16:00,967 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/A in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:16:00,967 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/4bcd44b755f84ac59e19ea9638ef3e71, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/662831dc07e94f498f5ea121f71e9de9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/32825a92e6a44126b801907c9c224d3e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/4aced2d8282c4e29a3ec7bb3ff410b5f] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=48.8 K 2024-11-15T23:16:00,967 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ef58bedac4ce437da383e1c2d90d9349, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/46b8ba7c2ae7467b95e1e164b8885a0b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/f95e1da0ef4b40aa98f7558f7df72822, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0be5e6495eb44eb1b5b29de7248f5b55] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=148.3 K 2024-11-15T23:16:00,967 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:16:00,967 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. files: [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ef58bedac4ce437da383e1c2d90d9349, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/46b8ba7c2ae7467b95e1e164b8885a0b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/f95e1da0ef4b40aa98f7558f7df72822, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0be5e6495eb44eb1b5b29de7248f5b55] 2024-11-15T23:16:00,968 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef58bedac4ce437da383e1c2d90d9349, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731712556015 2024-11-15T23:16:00,968 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bcd44b755f84ac59e19ea9638ef3e71, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731712556015 2024-11-15T23:16:00,970 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46b8ba7c2ae7467b95e1e164b8885a0b, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731712557236 2024-11-15T23:16:00,970 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 662831dc07e94f498f5ea121f71e9de9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731712557236 2024-11-15T23:16:00,970 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 32825a92e6a44126b801907c9c224d3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731712558534 2024-11-15T23:16:00,970 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting f95e1da0ef4b40aa98f7558f7df72822, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731712558534 2024-11-15T23:16:00,970 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aced2d8282c4e29a3ec7bb3ff410b5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731712559717 2024-11-15T23:16:00,970 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0be5e6495eb44eb1b5b29de7248f5b55, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731712559716 2024-11-15T23:16:00,980 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#B#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:00,980 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/a390b500ef994afea58baecaa0ad1a0b is 50, key is test_row_0/B:col10/1731712559717/Put/seqid=0 2024-11-15T23:16:00,981 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:16:00,983 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115b1ee04b349e94d19a113cf45e1a3b3ea_b3f0f65ece63912349255b0111be7183 store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:16:00,986 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115b1ee04b349e94d19a113cf45e1a3b3ea_b3f0f65ece63912349255b0111be7183, store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:16:00,986 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b1ee04b349e94d19a113cf45e1a3b3ea_b3f0f65ece63912349255b0111be7183 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:16:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:00,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T23:16:00,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:16:00,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:00,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:16:00,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:00,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:16:00,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:00,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742045_1221 (size=13153) 2024-11-15T23:16:00,999 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/a390b500ef994afea58baecaa0ad1a0b as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/a390b500ef994afea58baecaa0ad1a0b 2024-11-15T23:16:01,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ffca256080fb437d99a176cbf6e69248_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712560988/Put/seqid=0 2024-11-15T23:16:01,010 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b3f0f65ece63912349255b0111be7183/B of b3f0f65ece63912349255b0111be7183 into a390b500ef994afea58baecaa0ad1a0b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:01,010 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:16:01,010 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/B, priority=12, startTime=1731712560965; duration=0sec 2024-11-15T23:16:01,011 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:01,011 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:B 2024-11-15T23:16:01,011 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:16:01,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742046_1222 (size=4469) 2024-11-15T23:16:01,017 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:16:01,017 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): b3f0f65ece63912349255b0111be7183/C is initiating minor compaction (all files) 2024-11-15T23:16:01,017 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3f0f65ece63912349255b0111be7183/C in TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:16:01,018 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/b5e72eb9d06e436ca557f73594d19927, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/8150d02f6e01488dbcfe684fb278945e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/e88cf70a8a8c43b6a14ba71a4d3d9416, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a0f7856ccb234ced92d6b7f83316fd39] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp, totalSize=48.8 K 2024-11-15T23:16:01,018 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#A#compaction#181 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:01,018 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/b77e7e1429d74dfcb52cd214af0907fe is 175, key is test_row_0/A:col10/1731712559717/Put/seqid=0 2024-11-15T23:16:01,018 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting b5e72eb9d06e436ca557f73594d19927, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731712556015 2024-11-15T23:16:01,019 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 8150d02f6e01488dbcfe684fb278945e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731712557236 2024-11-15T23:16:01,019 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting e88cf70a8a8c43b6a14ba71a4d3d9416, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731712558534 2024-11-15T23:16:01,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742047_1223 (size=14994) 2024-11-15T23:16:01,020 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting a0f7856ccb234ced92d6b7f83316fd39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731712559717 2024-11-15T23:16:01,021 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:01,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742048_1224 (size=32107) 2024-11-15T23:16:01,026 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ffca256080fb437d99a176cbf6e69248_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ffca256080fb437d99a176cbf6e69248_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:01,027 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/23b1c8462edd440f8d4c02a5189104a7, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:16:01,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/23b1c8462edd440f8d4c02a5189104a7 is 175, key is test_row_0/A:col10/1731712560988/Put/seqid=0 2024-11-15T23:16:01,030 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/b77e7e1429d74dfcb52cd214af0907fe as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/b77e7e1429d74dfcb52cd214af0907fe 2024-11-15T23:16:01,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742049_1225 (size=39949) 2024-11-15T23:16:01,033 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3f0f65ece63912349255b0111be7183#C#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:01,033 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/a947c62f45a04101ba9d0735edce05d5 is 50, key is test_row_0/C:col10/1731712559717/Put/seqid=0 2024-11-15T23:16:01,034 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=401, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/23b1c8462edd440f8d4c02a5189104a7 2024-11-15T23:16:01,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742050_1226 (size=13153) 2024-11-15T23:16:01,039 INFO [Thread-625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-625. 2024-11-15T23:16:01,039 DEBUG [Thread-625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:16:01,039 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,040 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,040 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:01,041 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b3f0f65ece63912349255b0111be7183/A of b3f0f65ece63912349255b0111be7183 into b77e7e1429d74dfcb52cd214af0907fe(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:01,041 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:16:01,041 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/A, priority=12, startTime=1731712560965; duration=0sec 2024-11-15T23:16:01,041 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:01,041 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:A 2024-11-15T23:16:01,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/07a423f141cd488aa417c405ef918065 is 50, key is test_row_0/B:col10/1731712560988/Put/seqid=0 2024-11-15T23:16:01,045 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-11-15T23:16:01,045 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:16:01,045 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,045 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:01,045 INFO [Thread-631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-631. 2024-11-15T23:16:01,045 DEBUG [Thread-631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:16:01,045 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,045 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:01,048 INFO [Thread-616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-616. 2024-11-15T23:16:01,048 DEBUG [Thread-616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:16:01,048 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,048 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,048 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:01,048 INFO [Thread-627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-627. 2024-11-15T23:16:01,048 DEBUG [Thread-627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:16:01,048 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,048 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,048 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:01,049 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/a947c62f45a04101ba9d0735edce05d5 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a947c62f45a04101ba9d0735edce05d5 2024-11-15T23:16:01,049 INFO [Thread-629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-629. 2024-11-15T23:16:01,049 DEBUG [Thread-629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:16:01,049 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,049 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,050 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:01,052 INFO [Thread-620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-620. 2024-11-15T23:16:01,052 DEBUG [Thread-620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:16:01,052 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,052 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,052 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:01,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731712561027, max timestamp=1731712561047Num uniques 859; 2024-11-15T23:16:01,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:16:01,054 INFO [Thread-618 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-618. 2024-11-15T23:16:01,054 DEBUG [Thread-618 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:16:01,054 DEBUG [Thread-618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,055 DEBUG [Thread-618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:01,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712561027, max timestamp=1731712561047Num uniques 859; 2024-11-15T23:16:01,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:16:01,057 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b3f0f65ece63912349255b0111be7183/C of b3f0f65ece63912349255b0111be7183 into a947c62f45a04101ba9d0735edce05d5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:01,057 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:16:01,057 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., storeName=b3f0f65ece63912349255b0111be7183/C, priority=12, startTime=1731712560966; duration=0sec 2024-11-15T23:16:01,057 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:01,057 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3f0f65ece63912349255b0111be7183:C 2024-11-15T23:16:01,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742051_1227 (size=12301) 2024-11-15T23:16:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T23:16:01,417 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:01,439 INFO [Thread-614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-614. 2024-11-15T23:16:01,439 DEBUG [Thread-614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:16:01,439 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,439 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,439 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:01,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T23:16:01,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 131 2024-11-15T23:16:01,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-15T23:16:01,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-15T23:16:01,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-15T23:16:01,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-15T23:16:01,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T23:16:01,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2791 2024-11-15T23:16:01,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2662 2024-11-15T23:16:01,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T23:16:01,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1222 2024-11-15T23:16:01,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3665 rows 2024-11-15T23:16:01,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1220 2024-11-15T23:16:01,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3657 rows 2024-11-15T23:16:01,441 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T23:16:01,441 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T23:16:01,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:01,441 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:01,443 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T23:16:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T23:16:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T23:16:01,450 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712561450"}]},"ts":"1731712561450"} 2024-11-15T23:16:01,453 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T23:16:01,453 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T23:16:01,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T23:16:01,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3f0f65ece63912349255b0111be7183, UNASSIGN}] 2024-11-15T23:16:01,456 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3f0f65ece63912349255b0111be7183, UNASSIGN 2024-11-15T23:16:01,457 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=b3f0f65ece63912349255b0111be7183, regionState=CLOSING, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:01,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3f0f65ece63912349255b0111be7183, UNASSIGN because future has completed 2024-11-15T23:16:01,459 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T23:16:01,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763}] 2024-11-15T23:16:01,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/07a423f141cd488aa417c405ef918065 2024-11-15T23:16:01,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/47eb70e5e1f748acbe5cd1fd5b495f49 is 50, key is test_row_0/C:col10/1731712560988/Put/seqid=0 2024-11-15T23:16:01,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742052_1228 (size=12301) 2024-11-15T23:16:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T23:16:01,610 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:01,610 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T23:16:01,611 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing b3f0f65ece63912349255b0111be7183, disabling compactions & flushes 2024-11-15T23:16:01,611 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:16:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T23:16:01,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/47eb70e5e1f748acbe5cd1fd5b495f49 2024-11-15T23:16:01,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/23b1c8462edd440f8d4c02a5189104a7 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/23b1c8462edd440f8d4c02a5189104a7 2024-11-15T23:16:01,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/23b1c8462edd440f8d4c02a5189104a7, entries=200, sequenceid=401, filesize=39.0 K 2024-11-15T23:16:01,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/07a423f141cd488aa417c405ef918065 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/07a423f141cd488aa417c405ef918065 2024-11-15T23:16:01,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/07a423f141cd488aa417c405ef918065, entries=150, sequenceid=401, filesize=12.0 K 2024-11-15T23:16:01,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/47eb70e5e1f748acbe5cd1fd5b495f49 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/47eb70e5e1f748acbe5cd1fd5b495f49 2024-11-15T23:16:01,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/47eb70e5e1f748acbe5cd1fd5b495f49, entries=150, sequenceid=401, filesize=12.0 K 2024-11-15T23:16:01,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=67.09 KB/68700 for b3f0f65ece63912349255b0111be7183 in 919ms, sequenceid=401, compaction requested=false 2024-11-15T23:16:01,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3f0f65ece63912349255b0111be7183: 2024-11-15T23:16:01,908 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:16:01,908 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:16:01,908 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. after waiting 0 ms 2024-11-15T23:16:01,908 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:16:01,908 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing b3f0f65ece63912349255b0111be7183 3/3 column families, dataSize=67.09 KB heapSize=189.17 KB 2024-11-15T23:16:01,908 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=A 2024-11-15T23:16:01,908 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:01,908 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=B 2024-11-15T23:16:01,909 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:01,909 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b3f0f65ece63912349255b0111be7183, store=C 2024-11-15T23:16:01,909 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:01,914 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a2027e82da1843f293c40ad90f5fbf93_b3f0f65ece63912349255b0111be7183 is 50, key is test_row_0/A:col10/1731712561043/Put/seqid=0 2024-11-15T23:16:01,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742053_1229 (size=12454) 2024-11-15T23:16:01,967 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T23:16:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T23:16:02,320 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:02,330 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a2027e82da1843f293c40ad90f5fbf93_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a2027e82da1843f293c40ad90f5fbf93_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:02,330 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/d6d6ee3a7a354f5bb605c8c05143a193, store: [table=TestAcidGuarantees family=A region=b3f0f65ece63912349255b0111be7183] 2024-11-15T23:16:02,331 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/d6d6ee3a7a354f5bb605c8c05143a193 is 175, key is test_row_0/A:col10/1731712561043/Put/seqid=0 2024-11-15T23:16:02,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742054_1230 (size=31255) 2024-11-15T23:16:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T23:16:02,736 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=426, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/d6d6ee3a7a354f5bb605c8c05143a193 2024-11-15T23:16:02,753 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/a556e84d152e4c9c9ba6242592045023 is 50, key is test_row_0/B:col10/1731712561043/Put/seqid=0 2024-11-15T23:16:02,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742055_1231 (size=12301) 2024-11-15T23:16:03,157 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/a556e84d152e4c9c9ba6242592045023 2024-11-15T23:16:03,165 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/97eb2e8bfe0a4de395431c51a741c2d0 is 50, key is test_row_0/C:col10/1731712561043/Put/seqid=0 2024-11-15T23:16:03,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742056_1232 (size=12301) 2024-11-15T23:16:03,571 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/97eb2e8bfe0a4de395431c51a741c2d0 2024-11-15T23:16:03,579 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/A/d6d6ee3a7a354f5bb605c8c05143a193 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/d6d6ee3a7a354f5bb605c8c05143a193 2024-11-15T23:16:03,584 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/d6d6ee3a7a354f5bb605c8c05143a193, entries=150, sequenceid=426, filesize=30.5 K 2024-11-15T23:16:03,585 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/B/a556e84d152e4c9c9ba6242592045023 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/a556e84d152e4c9c9ba6242592045023 2024-11-15T23:16:03,589 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/a556e84d152e4c9c9ba6242592045023, entries=150, sequenceid=426, filesize=12.0 K 2024-11-15T23:16:03,590 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/.tmp/C/97eb2e8bfe0a4de395431c51a741c2d0 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/97eb2e8bfe0a4de395431c51a741c2d0 2024-11-15T23:16:03,594 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/97eb2e8bfe0a4de395431c51a741c2d0, entries=150, sequenceid=426, filesize=12.0 K 2024-11-15T23:16:03,595 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~189.13 KB/193664, currentSize=0 B/0 for b3f0f65ece63912349255b0111be7183 in 1687ms, sequenceid=426, compaction requested=true 2024-11-15T23:16:03,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/7d3c2ad948564cbca7321f77945ed1d4, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ff64021d15d94656bf11f7a187a3efe4, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/6e82ffdc266e4f6fa5e9053a7b71dc15, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ea99a96f7ea042cf9568451a86d4dc8a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/9d85301e07e14a5e86e2fde43ce5f6da, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/49eb0aa43a8d4240b7cdf718773d3781, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/49d64ca5d11e4b7490347b5aafd635c6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/cdc2239ef96d4759b5201adfafd749c7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ca2400dae3284f26abf2ceb6344e0569, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/6ba81febd8a84fa380412b523495b25a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/26e42a2608884fc4b3fa39cecfb35453, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0774813748d748018bfe15de9a56d0e6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/4a2e80ea953f4be9a3268555a2c4dd23, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0924bbaa8e5e48d3b48cff7354e6f9ce, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/f47cb2b1252c46fd9d46319556631e18, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/aad8b02cba5c43a086b9fe79862876a2, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/95231691bcb648a6869c888b4eabb08f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/862ba4ae2ea54277a2279be291271fb0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/11ea646fa195481d8a55215ba64962d5, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/fcf428ac917b45748aa3830b8179b29f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/c6239ecf05b94a059e4d4f69d5031d25, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ef58bedac4ce437da383e1c2d90d9349, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/46b8ba7c2ae7467b95e1e164b8885a0b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/f95e1da0ef4b40aa98f7558f7df72822, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0be5e6495eb44eb1b5b29de7248f5b55] to archive 2024-11-15T23:16:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T23:16:03,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:16:03,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/7d3c2ad948564cbca7321f77945ed1d4 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/7d3c2ad948564cbca7321f77945ed1d4 2024-11-15T23:16:03,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ff64021d15d94656bf11f7a187a3efe4 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ff64021d15d94656bf11f7a187a3efe4 2024-11-15T23:16:03,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/6e82ffdc266e4f6fa5e9053a7b71dc15 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/6e82ffdc266e4f6fa5e9053a7b71dc15 2024-11-15T23:16:03,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ea99a96f7ea042cf9568451a86d4dc8a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ea99a96f7ea042cf9568451a86d4dc8a 2024-11-15T23:16:03,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/9d85301e07e14a5e86e2fde43ce5f6da to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/9d85301e07e14a5e86e2fde43ce5f6da 2024-11-15T23:16:03,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/49eb0aa43a8d4240b7cdf718773d3781 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/49eb0aa43a8d4240b7cdf718773d3781 2024-11-15T23:16:03,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/49d64ca5d11e4b7490347b5aafd635c6 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/49d64ca5d11e4b7490347b5aafd635c6 2024-11-15T23:16:03,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/cdc2239ef96d4759b5201adfafd749c7 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/cdc2239ef96d4759b5201adfafd749c7 2024-11-15T23:16:03,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ca2400dae3284f26abf2ceb6344e0569 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ca2400dae3284f26abf2ceb6344e0569 2024-11-15T23:16:03,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/6ba81febd8a84fa380412b523495b25a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/6ba81febd8a84fa380412b523495b25a 2024-11-15T23:16:03,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/26e42a2608884fc4b3fa39cecfb35453 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/26e42a2608884fc4b3fa39cecfb35453 2024-11-15T23:16:03,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0774813748d748018bfe15de9a56d0e6 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0774813748d748018bfe15de9a56d0e6 2024-11-15T23:16:03,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/4a2e80ea953f4be9a3268555a2c4dd23 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/4a2e80ea953f4be9a3268555a2c4dd23 2024-11-15T23:16:03,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0924bbaa8e5e48d3b48cff7354e6f9ce to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0924bbaa8e5e48d3b48cff7354e6f9ce 2024-11-15T23:16:03,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/f47cb2b1252c46fd9d46319556631e18 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/f47cb2b1252c46fd9d46319556631e18 2024-11-15T23:16:03,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/aad8b02cba5c43a086b9fe79862876a2 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/aad8b02cba5c43a086b9fe79862876a2 2024-11-15T23:16:03,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/95231691bcb648a6869c888b4eabb08f to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/95231691bcb648a6869c888b4eabb08f 2024-11-15T23:16:03,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/862ba4ae2ea54277a2279be291271fb0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/862ba4ae2ea54277a2279be291271fb0 2024-11-15T23:16:03,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/11ea646fa195481d8a55215ba64962d5 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/11ea646fa195481d8a55215ba64962d5 2024-11-15T23:16:03,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/fcf428ac917b45748aa3830b8179b29f to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/fcf428ac917b45748aa3830b8179b29f 2024-11-15T23:16:03,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/c6239ecf05b94a059e4d4f69d5031d25 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/c6239ecf05b94a059e4d4f69d5031d25 2024-11-15T23:16:03,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ef58bedac4ce437da383e1c2d90d9349 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/ef58bedac4ce437da383e1c2d90d9349 2024-11-15T23:16:03,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/46b8ba7c2ae7467b95e1e164b8885a0b to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/46b8ba7c2ae7467b95e1e164b8885a0b 2024-11-15T23:16:03,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/f95e1da0ef4b40aa98f7558f7df72822 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/f95e1da0ef4b40aa98f7558f7df72822 2024-11-15T23:16:03,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0be5e6495eb44eb1b5b29de7248f5b55 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/0be5e6495eb44eb1b5b29de7248f5b55 2024-11-15T23:16:03,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/e05afa2d4b9847539849bdc8109a9c87, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/cd90d3bcff0d4e15bcef172f2bf626bb, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/77fa0cb42caa45c6a3ba9371d2318bcb, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/140fc76b84d847aca6a3413545a9b16f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/a01bbb26a9894547a1804022331bd91e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/3cf3b4848e98440a901094a233790239, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/1d90c0b0aae14c658071ef2c7e2f06c2, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/5b41f246bcb34a9ea24bf587f519b0f7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/2fcf0bf87b824156be0bbca63f049600, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/b0bfc55e45f74a3e8f4e147b5ba79513, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/7e056ce2f391408ea79f5010aad985cd, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/1e4aaa9d946441aab04e30d10a9a5d75, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/348eb998741142f990ff869fdec31316, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/49f0341709ae49e4bf0b65e90b833456, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/d2755d8eeb874f49b20a814391c75758, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/d4a2c76d58424049b150411bde65e76c, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/d0fdaae61dfe4a98ab8084adf6abbffc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/97b415be228e48fa83afe2f8d5b217c4, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/62f74adf351c47a186690beee575d90e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/4ebc2e7bed354a6a8917a6d9a122bebb, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/4bcd44b755f84ac59e19ea9638ef3e71, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/9a9d1ffb39be4c968e2da3ee6b6345d7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/662831dc07e94f498f5ea121f71e9de9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/32825a92e6a44126b801907c9c224d3e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/4aced2d8282c4e29a3ec7bb3ff410b5f] to archive 2024-11-15T23:16:03,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:16:03,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/e05afa2d4b9847539849bdc8109a9c87 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/e05afa2d4b9847539849bdc8109a9c87 2024-11-15T23:16:03,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/cd90d3bcff0d4e15bcef172f2bf626bb to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/cd90d3bcff0d4e15bcef172f2bf626bb 2024-11-15T23:16:03,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/77fa0cb42caa45c6a3ba9371d2318bcb to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/77fa0cb42caa45c6a3ba9371d2318bcb 2024-11-15T23:16:03,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/140fc76b84d847aca6a3413545a9b16f to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/140fc76b84d847aca6a3413545a9b16f 2024-11-15T23:16:03,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/a01bbb26a9894547a1804022331bd91e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/a01bbb26a9894547a1804022331bd91e 2024-11-15T23:16:03,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/3cf3b4848e98440a901094a233790239 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/3cf3b4848e98440a901094a233790239 2024-11-15T23:16:03,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/1d90c0b0aae14c658071ef2c7e2f06c2 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/1d90c0b0aae14c658071ef2c7e2f06c2 2024-11-15T23:16:03,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/5b41f246bcb34a9ea24bf587f519b0f7 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/5b41f246bcb34a9ea24bf587f519b0f7 2024-11-15T23:16:03,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/2fcf0bf87b824156be0bbca63f049600 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/2fcf0bf87b824156be0bbca63f049600 2024-11-15T23:16:03,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/b0bfc55e45f74a3e8f4e147b5ba79513 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/b0bfc55e45f74a3e8f4e147b5ba79513 2024-11-15T23:16:03,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/7e056ce2f391408ea79f5010aad985cd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/7e056ce2f391408ea79f5010aad985cd 2024-11-15T23:16:03,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/1e4aaa9d946441aab04e30d10a9a5d75 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/1e4aaa9d946441aab04e30d10a9a5d75 2024-11-15T23:16:03,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/348eb998741142f990ff869fdec31316 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/348eb998741142f990ff869fdec31316 2024-11-15T23:16:03,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/49f0341709ae49e4bf0b65e90b833456 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/49f0341709ae49e4bf0b65e90b833456 2024-11-15T23:16:03,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/d2755d8eeb874f49b20a814391c75758 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/d2755d8eeb874f49b20a814391c75758 2024-11-15T23:16:03,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/d4a2c76d58424049b150411bde65e76c to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/d4a2c76d58424049b150411bde65e76c 2024-11-15T23:16:03,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/d0fdaae61dfe4a98ab8084adf6abbffc to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/d0fdaae61dfe4a98ab8084adf6abbffc 2024-11-15T23:16:03,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/97b415be228e48fa83afe2f8d5b217c4 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/97b415be228e48fa83afe2f8d5b217c4 2024-11-15T23:16:03,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/62f74adf351c47a186690beee575d90e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/62f74adf351c47a186690beee575d90e 2024-11-15T23:16:03,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/4ebc2e7bed354a6a8917a6d9a122bebb to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/4ebc2e7bed354a6a8917a6d9a122bebb 2024-11-15T23:16:03,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/4bcd44b755f84ac59e19ea9638ef3e71 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/4bcd44b755f84ac59e19ea9638ef3e71 2024-11-15T23:16:03,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/9a9d1ffb39be4c968e2da3ee6b6345d7 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/9a9d1ffb39be4c968e2da3ee6b6345d7 2024-11-15T23:16:03,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/662831dc07e94f498f5ea121f71e9de9 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/662831dc07e94f498f5ea121f71e9de9 2024-11-15T23:16:03,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/32825a92e6a44126b801907c9c224d3e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/32825a92e6a44126b801907c9c224d3e 2024-11-15T23:16:03,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/4aced2d8282c4e29a3ec7bb3ff410b5f to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/4aced2d8282c4e29a3ec7bb3ff410b5f 2024-11-15T23:16:03,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/bb61e4cc2a634ad7a39d4b80ac5d1d47, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/5de54f3301154706a51c5645bc59bba5, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/ffa7752c5ead41c3967263cc8c62de28, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/24a9207601ca4d06b3f708e3baddcb38, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/1791ac203daf47039e4cac355046566d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/d8abdfb2eba0418eb1ec6bfce465f158, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/8d4d2df8791549ccbdd96a988028a839, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/4497c099f8294e379d181149a56398e8, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/19eceed266f14fa7b6492665f8eb4f04, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/b601a29d07b34103a20028b53bce1282, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/0ae47e4e1a164a999301c4e1c8eb0d7e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/e9311061c7e640769fd3a53b58225674, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/82cfed7be66743ca8f98ab45c3b0af4b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a3a1c567998f4634b1595e0b6e882fda, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/7a56cd6062b74fb2a69bc721c5ec82f3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a1c87af39835468fb6c2689f6f469eaf, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/ecd76d33651f4b25b5deac2f30fe0959, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a6986d71f3e9417395cad866ef9c700c, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a1ae3f616ec5439b96e5dc3d64783ece, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/b0ee33f751f840869c96832dae852167, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/b5e72eb9d06e436ca557f73594d19927, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/d1fe93065a4b4fd792d41400041de3b6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/8150d02f6e01488dbcfe684fb278945e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/e88cf70a8a8c43b6a14ba71a4d3d9416, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a0f7856ccb234ced92d6b7f83316fd39] to archive 2024-11-15T23:16:03,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:16:03,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/bb61e4cc2a634ad7a39d4b80ac5d1d47 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/bb61e4cc2a634ad7a39d4b80ac5d1d47 2024-11-15T23:16:03,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/5de54f3301154706a51c5645bc59bba5 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/5de54f3301154706a51c5645bc59bba5 2024-11-15T23:16:03,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/ffa7752c5ead41c3967263cc8c62de28 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/ffa7752c5ead41c3967263cc8c62de28 2024-11-15T23:16:03,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/24a9207601ca4d06b3f708e3baddcb38 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/24a9207601ca4d06b3f708e3baddcb38 2024-11-15T23:16:03,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/1791ac203daf47039e4cac355046566d to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/1791ac203daf47039e4cac355046566d 2024-11-15T23:16:03,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/d8abdfb2eba0418eb1ec6bfce465f158 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/d8abdfb2eba0418eb1ec6bfce465f158 2024-11-15T23:16:03,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/8d4d2df8791549ccbdd96a988028a839 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/8d4d2df8791549ccbdd96a988028a839 2024-11-15T23:16:03,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/4497c099f8294e379d181149a56398e8 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/4497c099f8294e379d181149a56398e8 2024-11-15T23:16:03,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/19eceed266f14fa7b6492665f8eb4f04 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/19eceed266f14fa7b6492665f8eb4f04 2024-11-15T23:16:03,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/b601a29d07b34103a20028b53bce1282 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/b601a29d07b34103a20028b53bce1282 2024-11-15T23:16:03,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/0ae47e4e1a164a999301c4e1c8eb0d7e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/0ae47e4e1a164a999301c4e1c8eb0d7e 2024-11-15T23:16:03,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/e9311061c7e640769fd3a53b58225674 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/e9311061c7e640769fd3a53b58225674 2024-11-15T23:16:03,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/82cfed7be66743ca8f98ab45c3b0af4b to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/82cfed7be66743ca8f98ab45c3b0af4b 2024-11-15T23:16:03,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a3a1c567998f4634b1595e0b6e882fda to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a3a1c567998f4634b1595e0b6e882fda 2024-11-15T23:16:03,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/7a56cd6062b74fb2a69bc721c5ec82f3 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/7a56cd6062b74fb2a69bc721c5ec82f3 2024-11-15T23:16:03,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a1c87af39835468fb6c2689f6f469eaf to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a1c87af39835468fb6c2689f6f469eaf 2024-11-15T23:16:03,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/ecd76d33651f4b25b5deac2f30fe0959 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/ecd76d33651f4b25b5deac2f30fe0959 2024-11-15T23:16:03,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a6986d71f3e9417395cad866ef9c700c to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a6986d71f3e9417395cad866ef9c700c 2024-11-15T23:16:03,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a1ae3f616ec5439b96e5dc3d64783ece to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a1ae3f616ec5439b96e5dc3d64783ece 2024-11-15T23:16:03,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/b0ee33f751f840869c96832dae852167 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/b0ee33f751f840869c96832dae852167 2024-11-15T23:16:03,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/b5e72eb9d06e436ca557f73594d19927 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/b5e72eb9d06e436ca557f73594d19927 2024-11-15T23:16:03,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/d1fe93065a4b4fd792d41400041de3b6 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/d1fe93065a4b4fd792d41400041de3b6 2024-11-15T23:16:03,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/8150d02f6e01488dbcfe684fb278945e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/8150d02f6e01488dbcfe684fb278945e 2024-11-15T23:16:03,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/e88cf70a8a8c43b6a14ba71a4d3d9416 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/e88cf70a8a8c43b6a14ba71a4d3d9416 2024-11-15T23:16:03,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a0f7856ccb234ced92d6b7f83316fd39 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a0f7856ccb234ced92d6b7f83316fd39 2024-11-15T23:16:03,687 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/recovered.edits/429.seqid, newMaxSeqId=429, maxSeqId=4 2024-11-15T23:16:03,688 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. 2024-11-15T23:16:03,688 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for b3f0f65ece63912349255b0111be7183: Waiting for close lock at 1731712561611Running coprocessor pre-close hooks at 1731712561611Disabling compacts and flushes for region at 1731712561611Disabling writes for close at 1731712561908 (+297 ms)Obtaining lock to block concurrent updates at 1731712561908Preparing flush snapshotting stores in b3f0f65ece63912349255b0111be7183 at 1731712561908Finished memstore snapshotting TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183., syncing WAL and waiting on mvcc, flushsize=dataSize=68700, getHeapSize=193664, getOffHeapSize=0, getCellsCount=1500 at 1731712561909 (+1 ms)Flushing stores of TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183. at 1731712561909Flushing b3f0f65ece63912349255b0111be7183/A: creating writer at 1731712561909Flushing mob file b3f0f65ece63912349255b0111be7183/A: appending metadata at 1731712561914 (+5 ms)Flushing mob file b3f0f65ece63912349255b0111be7183/A: closing flushed file at 1731712561914Flushing b3f0f65ece63912349255b0111be7183/A: appending metadata at 1731712562331 (+417 ms)Flushing b3f0f65ece63912349255b0111be7183/A: closing flushed file at 1731712562331Flushing b3f0f65ece63912349255b0111be7183/B: creating writer at 1731712562746 (+415 ms)Flushing b3f0f65ece63912349255b0111be7183/B: appending metadata at 1731712562753 (+7 ms)Flushing b3f0f65ece63912349255b0111be7183/B: closing flushed file at 1731712562753Flushing b3f0f65ece63912349255b0111be7183/C: creating writer at 1731712563162 (+409 ms)Flushing b3f0f65ece63912349255b0111be7183/C: appending metadata at 1731712563165 (+3 ms)Flushing b3f0f65ece63912349255b0111be7183/C: closing flushed file at 1731712563165Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13a85fb8: reopening flushed file at 1731712563578 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@317660a7: reopening flushed file at 1731712563584 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@660f36c9: reopening flushed file at 1731712563589 (+5 ms)Finished flush of dataSize ~67.09 KB/68700, heapSize ~189.13 KB/193664, currentSize=0 B/0 for b3f0f65ece63912349255b0111be7183 in 1687ms, sequenceid=426, compaction requested=true at 1731712563595 (+6 ms)Writing region close event to WAL at 1731712563684 (+89 ms)Running coprocessor post-close hooks at 1731712563688 (+4 ms)Closed at 1731712563688 2024-11-15T23:16:03,690 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:03,690 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=b3f0f65ece63912349255b0111be7183, regionState=CLOSED 2024-11-15T23:16:03,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 because future has completed 2024-11-15T23:16:03,694 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-15T23:16:03,694 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure b3f0f65ece63912349255b0111be7183, server=d6ecb9b7dda1,38353,1731712503763 in 2.2330 sec 2024-11-15T23:16:03,695 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-15T23:16:03,695 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3f0f65ece63912349255b0111be7183, UNASSIGN in 2.2390 sec 2024-11-15T23:16:03,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-15T23:16:03,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2430 sec 2024-11-15T23:16:03,699 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712563699"}]},"ts":"1731712563699"} 2024-11-15T23:16:03,701 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T23:16:03,701 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T23:16:03,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2580 sec 2024-11-15T23:16:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T23:16:05,606 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:05,606 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T23:16:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:16:05,608 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:16:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T23:16:05,609 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:16:05,611 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,613 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A, FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B, FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C, FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/recovered.edits] 2024-11-15T23:16:05,616 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/23b1c8462edd440f8d4c02a5189104a7 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/23b1c8462edd440f8d4c02a5189104a7 2024-11-15T23:16:05,617 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/b77e7e1429d74dfcb52cd214af0907fe to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/b77e7e1429d74dfcb52cd214af0907fe 2024-11-15T23:16:05,618 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/d6d6ee3a7a354f5bb605c8c05143a193 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/A/d6d6ee3a7a354f5bb605c8c05143a193 2024-11-15T23:16:05,621 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/07a423f141cd488aa417c405ef918065 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/07a423f141cd488aa417c405ef918065 2024-11-15T23:16:05,622 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/a390b500ef994afea58baecaa0ad1a0b to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/a390b500ef994afea58baecaa0ad1a0b 2024-11-15T23:16:05,623 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/a556e84d152e4c9c9ba6242592045023 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/B/a556e84d152e4c9c9ba6242592045023 2024-11-15T23:16:05,626 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/47eb70e5e1f748acbe5cd1fd5b495f49 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/47eb70e5e1f748acbe5cd1fd5b495f49 2024-11-15T23:16:05,627 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/97eb2e8bfe0a4de395431c51a741c2d0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/97eb2e8bfe0a4de395431c51a741c2d0 2024-11-15T23:16:05,628 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a947c62f45a04101ba9d0735edce05d5 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/C/a947c62f45a04101ba9d0735edce05d5 2024-11-15T23:16:05,630 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/recovered.edits/429.seqid to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183/recovered.edits/429.seqid 2024-11-15T23:16:05,630 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,630 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T23:16:05,630 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T23:16:05,631 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T23:16:05,635 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150949da2153f34eabb1d0dd98e45332cb_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150949da2153f34eabb1d0dd98e45332cb_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,637 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111512460595bf0848a582acd54c6a2736bc_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111512460595bf0848a582acd54c6a2736bc_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,638 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151266a4924cfa4df7abe4631b4739129c_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151266a4924cfa4df7abe4631b4739129c_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,639 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111520d661ed2c484c6dabe1375f2b6fe5bb_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111520d661ed2c484c6dabe1375f2b6fe5bb_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,640 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153ecee291601c4887bed642e15ebce1b1_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153ecee291601c4887bed642e15ebce1b1_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,642 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153f6cae7cb648440cb25a45785ef16dda_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153f6cae7cb648440cb25a45785ef16dda_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,643 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155962361fabf84e63a7820da4153e0110_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155962361fabf84e63a7820da4153e0110_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,644 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111560952cf0b3f34a4a8ffd800318f14cfd_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111560952cf0b3f34a4a8ffd800318f14cfd_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,645 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156e714c64263b4f098d6b1c4302365c03_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156e714c64263b4f098d6b1c4302365c03_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,646 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111577450401c6644509a2f3a561d9206b5f_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111577450401c6644509a2f3a561d9206b5f_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,647 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159c57a3b2cd29432a9c768b392ef15041_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159c57a3b2cd29432a9c768b392ef15041_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,649 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a0ccdc3f2da04210878835673fad6e8b_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a0ccdc3f2da04210878835673fad6e8b_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,650 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a2027e82da1843f293c40ad90f5fbf93_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a2027e82da1843f293c40ad90f5fbf93_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,651 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b0d1dde4cc434e5fa160a5f15774181f_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b0d1dde4cc434e5fa160a5f15774181f_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,652 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b4692d79f44b415eaed7497479d7bae9_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b4692d79f44b415eaed7497479d7bae9_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,654 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c24f70a5c34b49cbb7048b2350ce9953_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c24f70a5c34b49cbb7048b2350ce9953_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,654 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e65bd466c5114efe87a4afcce03258ad_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e65bd466c5114efe87a4afcce03258ad_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,655 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ed9edb89f48340f5bd3f7a23c2dc1376_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ed9edb89f48340f5bd3f7a23c2dc1376_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,656 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ef719ce087824e7ebf3ccae736b8b5e3_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ef719ce087824e7ebf3ccae736b8b5e3_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,657 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fed89b4ed87342f49b8c26b099fb65f3_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fed89b4ed87342f49b8c26b099fb65f3_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,658 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ffca256080fb437d99a176cbf6e69248_b3f0f65ece63912349255b0111be7183 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ffca256080fb437d99a176cbf6e69248_b3f0f65ece63912349255b0111be7183 2024-11-15T23:16:05,659 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T23:16:05,661 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:16:05,663 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T23:16:05,665 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T23:16:05,666 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:16:05,666 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T23:16:05,667 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731712565666"}]},"ts":"9223372036854775807"} 2024-11-15T23:16:05,669 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T23:16:05,669 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b3f0f65ece63912349255b0111be7183, NAME => 'TestAcidGuarantees,,1731712537850.b3f0f65ece63912349255b0111be7183.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T23:16:05,669 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T23:16:05,669 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731712565669"}]},"ts":"9223372036854775807"} 2024-11-15T23:16:05,671 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T23:16:05,672 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:16:05,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 66 msec 2024-11-15T23:16:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T23:16:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T23:16:05,716 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:05,728 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 248) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/cluster_535d8ff1-f3e7-a1c2-84cc-d10f4f48cd95/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/cluster_535d8ff1-f3e7-a1c2-84cc-d10f4f48cd95/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1925805657_22 at /127.0.0.1:33028 [Waiting for operation #227] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1603024157_22 at /127.0.0.1:33730 [Waiting for operation #135] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1603024157_22 at /127.0.0.1:32908 [Waiting for operation #262] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1925805657_22 at /127.0.0.1:33716 [Waiting for operation #146] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=463 (was 467), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=531 (was 436) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6349 (was 5620) - AvailableMemoryMB LEAK? - 2024-11-15T23:16:05,738 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=531, ProcessCount=11, AvailableMemoryMB=6348 2024-11-15T23:16:05,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T23:16:05,740 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T23:16:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T23:16:05,742 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T23:16:05,742 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:05,742 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-15T23:16:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-15T23:16:05,743 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T23:16:05,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742057_1233 (size=1061) 2024-11-15T23:16:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-15T23:16:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-15T23:16:06,151 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444 2024-11-15T23:16:06,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742058_1234 (size=53) 2024-11-15T23:16:06,157 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:16:06,157 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 083d01f7dba258590a0a4db0eb54177d, disabling compactions & flushes 2024-11-15T23:16:06,157 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:06,157 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:06,157 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. after waiting 0 ms 2024-11-15T23:16:06,157 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:06,157 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:06,157 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 083d01f7dba258590a0a4db0eb54177d: Waiting for close lock at 1731712566157Disabling compacts and flushes for region at 1731712566157Disabling writes for close at 1731712566157Writing region close event to WAL at 1731712566157Closed at 1731712566157 2024-11-15T23:16:06,159 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T23:16:06,159 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731712566159"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731712566159"}]},"ts":"1731712566159"} 2024-11-15T23:16:06,160 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T23:16:06,162 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T23:16:06,162 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712566162"}]},"ts":"1731712566162"} 2024-11-15T23:16:06,163 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T23:16:06,163 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=083d01f7dba258590a0a4db0eb54177d, ASSIGN}] 2024-11-15T23:16:06,165 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=083d01f7dba258590a0a4db0eb54177d, ASSIGN 2024-11-15T23:16:06,165 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=083d01f7dba258590a0a4db0eb54177d, ASSIGN; state=OFFLINE, location=d6ecb9b7dda1,38353,1731712503763; forceNewPlan=false, retain=false 2024-11-15T23:16:06,316 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=083d01f7dba258590a0a4db0eb54177d, regionState=OPENING, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:06,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=083d01f7dba258590a0a4db0eb54177d, ASSIGN because future has completed 2024-11-15T23:16:06,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763}] 2024-11-15T23:16:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-15T23:16:06,475 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:06,475 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} 2024-11-15T23:16:06,475 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:06,476 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:16:06,476 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:06,476 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:06,478 INFO [StoreOpener-083d01f7dba258590a0a4db0eb54177d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:06,479 INFO [StoreOpener-083d01f7dba258590a0a4db0eb54177d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:16:06,480 INFO [StoreOpener-083d01f7dba258590a0a4db0eb54177d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 083d01f7dba258590a0a4db0eb54177d columnFamilyName A 2024-11-15T23:16:06,480 DEBUG [StoreOpener-083d01f7dba258590a0a4db0eb54177d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:06,480 INFO [StoreOpener-083d01f7dba258590a0a4db0eb54177d-1 {}] regionserver.HStore(327): Store=083d01f7dba258590a0a4db0eb54177d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:16:06,481 INFO [StoreOpener-083d01f7dba258590a0a4db0eb54177d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:06,482 INFO [StoreOpener-083d01f7dba258590a0a4db0eb54177d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:16:06,482 INFO [StoreOpener-083d01f7dba258590a0a4db0eb54177d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 083d01f7dba258590a0a4db0eb54177d columnFamilyName B 2024-11-15T23:16:06,482 DEBUG [StoreOpener-083d01f7dba258590a0a4db0eb54177d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:06,482 INFO [StoreOpener-083d01f7dba258590a0a4db0eb54177d-1 {}] regionserver.HStore(327): Store=083d01f7dba258590a0a4db0eb54177d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:16:06,482 INFO [StoreOpener-083d01f7dba258590a0a4db0eb54177d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:06,484 INFO [StoreOpener-083d01f7dba258590a0a4db0eb54177d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:16:06,484 INFO [StoreOpener-083d01f7dba258590a0a4db0eb54177d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 083d01f7dba258590a0a4db0eb54177d columnFamilyName C 2024-11-15T23:16:06,484 DEBUG [StoreOpener-083d01f7dba258590a0a4db0eb54177d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:06,485 INFO [StoreOpener-083d01f7dba258590a0a4db0eb54177d-1 {}] regionserver.HStore(327): Store=083d01f7dba258590a0a4db0eb54177d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:16:06,485 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:06,485 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:06,486 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:06,486 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:06,487 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:06,487 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:06,488 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T23:16:06,489 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:06,491 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T23:16:06,492 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 083d01f7dba258590a0a4db0eb54177d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66912190, jitterRate=-0.002930670976638794}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T23:16:06,492 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:06,493 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 083d01f7dba258590a0a4db0eb54177d: Running coprocessor pre-open hook at 1731712566476Writing region info on filesystem at 1731712566476Initializing all the Stores at 1731712566477 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712566477Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712566478 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712566478Cleaning up temporary data from old regions at 1731712566487 (+9 ms)Running coprocessor post-open hooks at 1731712566492 (+5 ms)Region opened successfully at 1731712566492 2024-11-15T23:16:06,493 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., pid=61, masterSystemTime=1731712566472 2024-11-15T23:16:06,495 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:06,495 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:06,496 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=083d01f7dba258590a0a4db0eb54177d, regionState=OPEN, openSeqNum=2, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:06,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 because future has completed 2024-11-15T23:16:06,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-15T23:16:06,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 in 179 msec 2024-11-15T23:16:06,501 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-15T23:16:06,501 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=083d01f7dba258590a0a4db0eb54177d, ASSIGN in 336 msec 2024-11-15T23:16:06,502 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T23:16:06,502 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712566502"}]},"ts":"1731712566502"} 2024-11-15T23:16:06,503 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T23:16:06,505 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T23:16:06,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 765 msec 2024-11-15T23:16:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-15T23:16:06,877 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:06,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a25e579, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,880 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:06,880 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:06,883 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:06,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:06,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:06,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e2b66e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:06,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:06,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:06,886 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:06,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79a86dc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,888 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:06,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:06,891 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T23:16:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is d6ecb9b7dda1,43421,1731712502855 2024-11-15T23:16:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5a90e563 2024-11-15T23:16:06,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T23:16:06,893 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T23:16:06,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,895 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:06,895 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:06,896 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37dd5517, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:06,897 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@398537a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,898 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:06,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:06,899 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:06,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:06,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:06,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d388952, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:06,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:06,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:06,901 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:06,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52d4c28c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:06,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:06,902 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:06,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:06,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23caed92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:06,903 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:06,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b0a3b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,904 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:06,904 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:06,905 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19ba0509, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:06,906 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:06,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ab460ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,907 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:06,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:06,908 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:06,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:06,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:06,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c130baf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:06,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:06,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:06,909 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:06,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71fd21f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:06,910 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:06,911 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:06,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:06,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:06,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5136ccb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:06,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:06,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:06,912 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:06,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ef6f83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:06,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:06,914 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11b9c7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:06,915 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:06,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5abc1b8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,916 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:06,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:06,917 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:06,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:06,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:06,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b58a0a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:06,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:06,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:06,918 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:06,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14741f05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:06,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:06,920 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:06,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:06,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:06,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ab132fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:06,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:06,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:06,921 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:06,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bcb9620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e1d0fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,922 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:06,922 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:06,922 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:06,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:06,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:06,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47cd5469, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:06,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:06,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:06,924 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:06,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dda9a20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:06,925 DEBUG [Thread-1114 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:06,925 DEBUG [Thread-1097 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:06,925 DEBUG [Thread-1103 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:06,925 DEBUG [Thread-1099 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:06,925 DEBUG [Thread-1105 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:06,925 DEBUG [Thread-1112 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:06,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:06,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:06,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:06,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:06,926 DEBUG [Thread-1110 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:06,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:06,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:06,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:06,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:06,928 DEBUG [Thread-1101 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:06,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:06,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:06,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:06,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:06,928 DEBUG [Thread-1108 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:06,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:06,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:06,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:06,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:16:06,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:16:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:06,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:06,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:06,933 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:16:06,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:16:06,933 DEBUG [Thread-1116 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:06,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:16:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-15T23:16:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:16:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:16:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:16:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T23:16:06,935 DEBUG [Thread-1101 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:06,935 DEBUG [Thread-1101 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:06,935 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:16:06,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:16:06,936 DEBUG [Thread-1116 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:06,936 DEBUG [Thread-1116 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:06,936 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:16:06,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:16:06,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:06,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:06,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:16:06,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:16:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:06,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T23:16:06,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:06,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:06,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:06,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:06,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:06,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:06,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/773a5d1b9b6b4e2c969d5223d22c617e is 50, key is test_row_0/A:col10/1731712566941/Put/seqid=0 2024-11-15T23:16:06,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712626977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:06,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712626979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:06,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712626979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:06,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712626980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:06,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712626981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:06,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742059_1235 (size=12001) 2024-11-15T23:16:06,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/773a5d1b9b6b4e2c969d5223d22c617e 2024-11-15T23:16:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T23:16:07,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/bcf65eb543654c32b723d33d7c40e9ba is 50, key is test_row_0/B:col10/1731712566941/Put/seqid=0 2024-11-15T23:16:07,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742060_1236 (size=12001) 2024-11-15T23:16:07,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/bcf65eb543654c32b723d33d7c40e9ba 2024-11-15T23:16:07,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712627086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T23:16:07,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:07,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:07,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:07,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:07,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:07,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712627095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712627095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712627096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712627096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/866ef9c2628e4dd991052ad8b18a07c1 is 50, key is test_row_0/C:col10/1731712566941/Put/seqid=0 2024-11-15T23:16:07,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742061_1237 (size=12001) 2024-11-15T23:16:07,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/866ef9c2628e4dd991052ad8b18a07c1 2024-11-15T23:16:07,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/773a5d1b9b6b4e2c969d5223d22c617e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/773a5d1b9b6b4e2c969d5223d22c617e 2024-11-15T23:16:07,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/773a5d1b9b6b4e2c969d5223d22c617e, entries=150, sequenceid=14, filesize=11.7 K 2024-11-15T23:16:07,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/bcf65eb543654c32b723d33d7c40e9ba as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/bcf65eb543654c32b723d33d7c40e9ba 2024-11-15T23:16:07,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/bcf65eb543654c32b723d33d7c40e9ba, entries=150, sequenceid=14, filesize=11.7 K 2024-11-15T23:16:07,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/866ef9c2628e4dd991052ad8b18a07c1 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/866ef9c2628e4dd991052ad8b18a07c1 2024-11-15T23:16:07,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/866ef9c2628e4dd991052ad8b18a07c1, entries=150, sequenceid=14, filesize=11.7 K 2024-11-15T23:16:07,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 083d01f7dba258590a0a4db0eb54177d in 268ms, sequenceid=14, compaction requested=false 2024-11-15T23:16:07,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:07,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T23:16:07,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:07,245 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T23:16:07,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:07,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:07,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:07,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:07,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:07,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:07,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/a2552639ad954511ba6d85dbf2e11b32 is 50, key is test_row_0/A:col10/1731712566979/Put/seqid=0 2024-11-15T23:16:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T23:16:07,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742062_1238 (size=12001) 2024-11-15T23:16:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:07,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:07,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712627313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712627334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712627337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712627337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712627338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712627446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712627446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712627446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712627446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712627446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T23:16:07,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712627655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712627655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712627655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712627655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712627655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,659 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/a2552639ad954511ba6d85dbf2e11b32 2024-11-15T23:16:07,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/302d8cdde9dc43b8874bfa7699f983fc is 50, key is test_row_0/B:col10/1731712566979/Put/seqid=0 2024-11-15T23:16:07,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742063_1239 (size=12001) 2024-11-15T23:16:07,671 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/302d8cdde9dc43b8874bfa7699f983fc 2024-11-15T23:16:07,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/b81f3c0d99c944b5ac11198ba95156bc is 50, key is test_row_0/C:col10/1731712566979/Put/seqid=0 2024-11-15T23:16:07,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742064_1240 (size=12001) 2024-11-15T23:16:07,682 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/b81f3c0d99c944b5ac11198ba95156bc 2024-11-15T23:16:07,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/a2552639ad954511ba6d85dbf2e11b32 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/a2552639ad954511ba6d85dbf2e11b32 2024-11-15T23:16:07,690 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/a2552639ad954511ba6d85dbf2e11b32, entries=150, sequenceid=33, filesize=11.7 K 2024-11-15T23:16:07,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/302d8cdde9dc43b8874bfa7699f983fc as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/302d8cdde9dc43b8874bfa7699f983fc 2024-11-15T23:16:07,695 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/302d8cdde9dc43b8874bfa7699f983fc, entries=150, sequenceid=33, filesize=11.7 K 2024-11-15T23:16:07,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/b81f3c0d99c944b5ac11198ba95156bc as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/b81f3c0d99c944b5ac11198ba95156bc 2024-11-15T23:16:07,701 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/b81f3c0d99c944b5ac11198ba95156bc, entries=150, sequenceid=33, filesize=11.7 K 2024-11-15T23:16:07,702 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=80.51 KB/82440 for 083d01f7dba258590a0a4db0eb54177d in 457ms, sequenceid=33, compaction requested=false 2024-11-15T23:16:07,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:07,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:07,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-15T23:16:07,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-15T23:16:07,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-15T23:16:07,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 767 msec 2024-11-15T23:16:07,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 773 msec 2024-11-15T23:16:07,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:07,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T23:16:07,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:07,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:07,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:07,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:07,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:07,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:07,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/8a26f4f97b6c4068b4b3fd3a33f14e21 is 50, key is test_row_0/A:col10/1731712567335/Put/seqid=0 2024-11-15T23:16:07,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742065_1241 (size=14341) 2024-11-15T23:16:07,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/8a26f4f97b6c4068b4b3fd3a33f14e21 2024-11-15T23:16:07,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712627982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712627983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712627985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:07,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712627987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712627988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/5a8367cc6366443fbf0203806ac7b052 is 50, key is test_row_0/B:col10/1731712567335/Put/seqid=0 2024-11-15T23:16:08,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742066_1242 (size=12001) 2024-11-15T23:16:08,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T23:16:08,075 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:08,076 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:16:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-15T23:16:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T23:16:08,078 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:16:08,078 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:16:08,078 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:16:08,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712628095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712628096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712628096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712628096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712628096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T23:16:08,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T23:16:08,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:08,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:08,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:08,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:08,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:08,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:08,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712628305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712628306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712628306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712628306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712628306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T23:16:08,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:08,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:08,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:08,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:08,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T23:16:08,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/5a8367cc6366443fbf0203806ac7b052 2024-11-15T23:16:08,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/343bd8548b244574838184fe8c82f839 is 50, key is test_row_0/C:col10/1731712567335/Put/seqid=0 2024-11-15T23:16:08,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742067_1243 (size=12001) 2024-11-15T23:16:08,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/343bd8548b244574838184fe8c82f839 2024-11-15T23:16:08,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/8a26f4f97b6c4068b4b3fd3a33f14e21 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/8a26f4f97b6c4068b4b3fd3a33f14e21 2024-11-15T23:16:08,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/8a26f4f97b6c4068b4b3fd3a33f14e21, entries=200, sequenceid=50, filesize=14.0 K 2024-11-15T23:16:08,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/5a8367cc6366443fbf0203806ac7b052 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/5a8367cc6366443fbf0203806ac7b052 2024-11-15T23:16:08,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/5a8367cc6366443fbf0203806ac7b052, entries=150, sequenceid=50, filesize=11.7 K 2024-11-15T23:16:08,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/343bd8548b244574838184fe8c82f839 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/343bd8548b244574838184fe8c82f839 2024-11-15T23:16:08,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/343bd8548b244574838184fe8c82f839, entries=150, sequenceid=50, filesize=11.7 K 2024-11-15T23:16:08,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 083d01f7dba258590a0a4db0eb54177d in 507ms, sequenceid=50, compaction requested=true 2024-11-15T23:16:08,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:08,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:16:08,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:08,476 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:08,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:16:08,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:08,476 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:08,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:16:08,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:08,477 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:08,477 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/B is initiating minor compaction (all files) 2024-11-15T23:16:08,477 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/B in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:08,477 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/bcf65eb543654c32b723d33d7c40e9ba, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/302d8cdde9dc43b8874bfa7699f983fc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/5a8367cc6366443fbf0203806ac7b052] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=35.2 K 2024-11-15T23:16:08,477 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:08,477 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/A is initiating minor compaction (all files) 2024-11-15T23:16:08,477 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/A in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:08,477 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/773a5d1b9b6b4e2c969d5223d22c617e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/a2552639ad954511ba6d85dbf2e11b32, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/8a26f4f97b6c4068b4b3fd3a33f14e21] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=37.4 K 2024-11-15T23:16:08,478 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 773a5d1b9b6b4e2c969d5223d22c617e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731712566939 2024-11-15T23:16:08,478 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting bcf65eb543654c32b723d33d7c40e9ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731712566939 2024-11-15T23:16:08,478 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2552639ad954511ba6d85dbf2e11b32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731712566977 2024-11-15T23:16:08,479 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 302d8cdde9dc43b8874bfa7699f983fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731712566977 2024-11-15T23:16:08,479 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a26f4f97b6c4068b4b3fd3a33f14e21, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731712567334 2024-11-15T23:16:08,480 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a8367cc6366443fbf0203806ac7b052, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731712567335 2024-11-15T23:16:08,489 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#B#compaction#199 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:08,489 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#A#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:08,489 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/68c38dc0c8ac44babff389795279fc51 is 50, key is test_row_0/B:col10/1731712567335/Put/seqid=0 2024-11-15T23:16:08,489 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/c53082910d1e4b76a571cfa5099e9721 is 50, key is test_row_0/A:col10/1731712567335/Put/seqid=0 2024-11-15T23:16:08,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742068_1244 (size=12104) 2024-11-15T23:16:08,503 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/c53082910d1e4b76a571cfa5099e9721 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/c53082910d1e4b76a571cfa5099e9721 2024-11-15T23:16:08,512 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/A of 083d01f7dba258590a0a4db0eb54177d into c53082910d1e4b76a571cfa5099e9721(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:08,512 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:08,512 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/A, priority=13, startTime=1731712568476; duration=0sec 2024-11-15T23:16:08,512 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:08,512 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:A 2024-11-15T23:16:08,512 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:08,514 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:08,514 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/C is initiating minor compaction (all files) 2024-11-15T23:16:08,514 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/C in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:08,514 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/866ef9c2628e4dd991052ad8b18a07c1, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/b81f3c0d99c944b5ac11198ba95156bc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/343bd8548b244574838184fe8c82f839] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=35.2 K 2024-11-15T23:16:08,515 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 866ef9c2628e4dd991052ad8b18a07c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731712566939 2024-11-15T23:16:08,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742069_1245 (size=12104) 2024-11-15T23:16:08,515 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting b81f3c0d99c944b5ac11198ba95156bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731712566977 2024-11-15T23:16:08,515 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 343bd8548b244574838184fe8c82f839, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731712567335 2024-11-15T23:16:08,526 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/68c38dc0c8ac44babff389795279fc51 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/68c38dc0c8ac44babff389795279fc51 2024-11-15T23:16:08,532 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#C#compaction#200 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:08,532 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/1eebfe359b884d6493535beafe9cede7 is 50, key is test_row_0/C:col10/1731712567335/Put/seqid=0 2024-11-15T23:16:08,534 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/B of 083d01f7dba258590a0a4db0eb54177d into 68c38dc0c8ac44babff389795279fc51(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:08,534 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:08,535 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/B, priority=13, startTime=1731712568476; duration=0sec 2024-11-15T23:16:08,535 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:08,535 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:B 2024-11-15T23:16:08,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T23:16:08,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:08,538 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T23:16:08,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:08,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:08,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:08,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:08,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:08,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:08,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742070_1246 (size=12104) 2024-11-15T23:16:08,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/80633ca5068d45548cdcb4be2f1d1517 is 50, key is test_row_0/A:col10/1731712567984/Put/seqid=0 2024-11-15T23:16:08,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742071_1247 (size=12001) 2024-11-15T23:16:08,600 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/80633ca5068d45548cdcb4be2f1d1517 2024-11-15T23:16:08,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:08,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:08,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/9eb9ace7c9054b32a2e9ad552cf0e5c3 is 50, key is test_row_0/B:col10/1731712567984/Put/seqid=0 2024-11-15T23:16:08,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742072_1248 (size=12001) 2024-11-15T23:16:08,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712628682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712628684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T23:16:08,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712628695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712628695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712628705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712628805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712628805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712628816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712628816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:08,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712628816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:08,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:08,965 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/1eebfe359b884d6493535beafe9cede7 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/1eebfe359b884d6493535beafe9cede7 2024-11-15T23:16:08,970 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/C of 083d01f7dba258590a0a4db0eb54177d into 1eebfe359b884d6493535beafe9cede7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:08,970 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:08,970 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/C, priority=13, startTime=1731712568476; duration=0sec 2024-11-15T23:16:08,970 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:08,970 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:C 2024-11-15T23:16:09,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712629016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712629016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:09,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712629025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712629025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712629025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:09,067 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/9eb9ace7c9054b32a2e9ad552cf0e5c3 2024-11-15T23:16:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/cad29a4b50ea44c7a6e377aeb67beb66 is 50, key is test_row_0/C:col10/1731712567984/Put/seqid=0 2024-11-15T23:16:09,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742073_1249 (size=12001) 2024-11-15T23:16:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T23:16:09,224 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T23:16:09,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712629325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712629325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:09,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712629335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712629335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:09,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712629336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:09,486 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/cad29a4b50ea44c7a6e377aeb67beb66 2024-11-15T23:16:09,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/80633ca5068d45548cdcb4be2f1d1517 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/80633ca5068d45548cdcb4be2f1d1517 2024-11-15T23:16:09,499 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/80633ca5068d45548cdcb4be2f1d1517, entries=150, sequenceid=68, filesize=11.7 K 2024-11-15T23:16:09,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/9eb9ace7c9054b32a2e9ad552cf0e5c3 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/9eb9ace7c9054b32a2e9ad552cf0e5c3 2024-11-15T23:16:09,506 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/9eb9ace7c9054b32a2e9ad552cf0e5c3, entries=150, sequenceid=68, filesize=11.7 K 2024-11-15T23:16:09,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/cad29a4b50ea44c7a6e377aeb67beb66 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/cad29a4b50ea44c7a6e377aeb67beb66 2024-11-15T23:16:09,514 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/cad29a4b50ea44c7a6e377aeb67beb66, entries=150, sequenceid=68, filesize=11.7 K 2024-11-15T23:16:09,516 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 083d01f7dba258590a0a4db0eb54177d in 978ms, sequenceid=68, compaction requested=false 2024-11-15T23:16:09,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:09,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:09,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-15T23:16:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-15T23:16:09,521 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-15T23:16:09,521 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4390 sec 2024-11-15T23:16:09,524 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.4460 sec 2024-11-15T23:16:09,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:09,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T23:16:09,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:09,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:09,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:09,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:09,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:09,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:09,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/c667bd939d064d559368abb8c6ed6708 is 50, key is test_row_0/A:col10/1731712569836/Put/seqid=0 2024-11-15T23:16:09,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742074_1250 (size=14341) 2024-11-15T23:16:09,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/c667bd939d064d559368abb8c6ed6708 2024-11-15T23:16:09,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/2813dfac7488406aaa2dfedfa443a923 is 50, key is test_row_0/B:col10/1731712569836/Put/seqid=0 2024-11-15T23:16:09,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712629852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:09,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712629857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712629858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:09,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712629858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:09,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742075_1251 (size=12001) 2024-11-15T23:16:09,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712629859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:09,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:09,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/2813dfac7488406aaa2dfedfa443a923 2024-11-15T23:16:09,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/1c7705176b9a43cbae955c75696bcaad is 50, key is test_row_0/C:col10/1731712569836/Put/seqid=0 2024-11-15T23:16:09,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742076_1252 (size=12001) 2024-11-15T23:16:09,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712629965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:09,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712629966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:09,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712629975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712629975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712629975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712630176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712630176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712630186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712630186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712630186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T23:16:10,225 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:10,225 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:16:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-15T23:16:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T23:16:10,227 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:16:10,228 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:16:10,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:16:10,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/1c7705176b9a43cbae955c75696bcaad 2024-11-15T23:16:10,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/c667bd939d064d559368abb8c6ed6708 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/c667bd939d064d559368abb8c6ed6708 2024-11-15T23:16:10,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/c667bd939d064d559368abb8c6ed6708, entries=200, sequenceid=87, filesize=14.0 K 2024-11-15T23:16:10,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/2813dfac7488406aaa2dfedfa443a923 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2813dfac7488406aaa2dfedfa443a923 2024-11-15T23:16:10,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2813dfac7488406aaa2dfedfa443a923, entries=150, sequenceid=87, filesize=11.7 K 2024-11-15T23:16:10,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/1c7705176b9a43cbae955c75696bcaad as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/1c7705176b9a43cbae955c75696bcaad 2024-11-15T23:16:10,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/1c7705176b9a43cbae955c75696bcaad, entries=150, sequenceid=87, filesize=11.7 K 2024-11-15T23:16:10,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 083d01f7dba258590a0a4db0eb54177d in 461ms, sequenceid=87, compaction requested=true 2024-11-15T23:16:10,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:16:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:10,299 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:16:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:10,299 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:16:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:10,300 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:10,300 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:10,300 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/A is initiating minor compaction (all files) 2024-11-15T23:16:10,300 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/B is initiating minor compaction (all files) 2024-11-15T23:16:10,300 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/B in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:10,300 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/A in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:10,300 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/c53082910d1e4b76a571cfa5099e9721, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/80633ca5068d45548cdcb4be2f1d1517, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/c667bd939d064d559368abb8c6ed6708] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=37.5 K 2024-11-15T23:16:10,300 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/68c38dc0c8ac44babff389795279fc51, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/9eb9ace7c9054b32a2e9ad552cf0e5c3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2813dfac7488406aaa2dfedfa443a923] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=35.3 K 2024-11-15T23:16:10,301 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 68c38dc0c8ac44babff389795279fc51, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731712567335 2024-11-15T23:16:10,301 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting c53082910d1e4b76a571cfa5099e9721, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731712567335 2024-11-15T23:16:10,302 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 9eb9ace7c9054b32a2e9ad552cf0e5c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731712567981 2024-11-15T23:16:10,302 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80633ca5068d45548cdcb4be2f1d1517, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731712567981 2024-11-15T23:16:10,303 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting c667bd939d064d559368abb8c6ed6708, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731712568680 2024-11-15T23:16:10,303 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 2813dfac7488406aaa2dfedfa443a923, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731712568685 2024-11-15T23:16:10,322 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#B#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:10,323 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/c283991331554e60a1f536235cf2d7b4 is 50, key is test_row_0/B:col10/1731712569836/Put/seqid=0 2024-11-15T23:16:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T23:16:10,339 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#A#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:10,340 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/a84f9cd8eacc448dbb8ed3537419b652 is 50, key is test_row_0/A:col10/1731712569836/Put/seqid=0 2024-11-15T23:16:10,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742077_1253 (size=12207) 2024-11-15T23:16:10,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742078_1254 (size=12207) 2024-11-15T23:16:10,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T23:16:10,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:10,380 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T23:16:10,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:10,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:10,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:10,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:10,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:10,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:10,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/52030626c2414c319b99a70f8b5dfb5e is 50, key is test_row_0/A:col10/1731712569857/Put/seqid=0 2024-11-15T23:16:10,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742079_1255 (size=12001) 2024-11-15T23:16:10,394 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/52030626c2414c319b99a70f8b5dfb5e 2024-11-15T23:16:10,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/2a7e4d42fba640dcaa7e8a6654be36a2 is 50, key is test_row_0/B:col10/1731712569857/Put/seqid=0 2024-11-15T23:16:10,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742080_1256 (size=12001) 2024-11-15T23:16:10,415 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/2a7e4d42fba640dcaa7e8a6654be36a2 2024-11-15T23:16:10,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/aabc507ac2744c93bd6e1560a606e775 is 50, key is test_row_0/C:col10/1731712569857/Put/seqid=0 2024-11-15T23:16:10,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742081_1257 (size=12001) 2024-11-15T23:16:10,429 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/aabc507ac2744c93bd6e1560a606e775 2024-11-15T23:16:10,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/52030626c2414c319b99a70f8b5dfb5e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/52030626c2414c319b99a70f8b5dfb5e 2024-11-15T23:16:10,440 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/52030626c2414c319b99a70f8b5dfb5e, entries=150, sequenceid=102, filesize=11.7 K 2024-11-15T23:16:10,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/2a7e4d42fba640dcaa7e8a6654be36a2 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2a7e4d42fba640dcaa7e8a6654be36a2 2024-11-15T23:16:10,445 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2a7e4d42fba640dcaa7e8a6654be36a2, entries=150, sequenceid=102, filesize=11.7 K 2024-11-15T23:16:10,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/aabc507ac2744c93bd6e1560a606e775 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/aabc507ac2744c93bd6e1560a606e775 2024-11-15T23:16:10,467 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/aabc507ac2744c93bd6e1560a606e775, entries=150, sequenceid=102, filesize=11.7 K 2024-11-15T23:16:10,468 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 083d01f7dba258590a0a4db0eb54177d in 88ms, sequenceid=102, compaction requested=true 2024-11-15T23:16:10,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:10,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:10,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-15T23:16:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-15T23:16:10,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-15T23:16:10,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 242 msec 2024-11-15T23:16:10,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 248 msec 2024-11-15T23:16:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:10,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T23:16:10,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:10,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:10,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:10,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:10,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:10,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:10,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/8762d1a9527347ae9b8e9cd44da25cbf is 50, key is test_row_1/A:col10/1731712570492/Put/seqid=0 2024-11-15T23:16:10,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742082_1258 (size=11997) 2024-11-15T23:16:10,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712630516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712570510, max timestamp=1731712570520Num uniques 859; 2024-11-15T23:16:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712630517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:16:10,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712570510, max timestamp=1731712570520Num uniques 859; 2024-11-15T23:16:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:16:10,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712570510, max timestamp=1731712570520Num uniques 859; 2024-11-15T23:16:10,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:16:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T23:16:10,545 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:10,546 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:16:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-15T23:16:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T23:16:10,547 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:16:10,548 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:16:10,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:16:10,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712630590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712630591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712630595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712630625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712630625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T23:16:10,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:10,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:10,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:10,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:10,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:10,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:10,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:10,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712630707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712630708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712630708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,771 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/c283991331554e60a1f536235cf2d7b4 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/c283991331554e60a1f536235cf2d7b4 2024-11-15T23:16:10,777 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/B of 083d01f7dba258590a0a4db0eb54177d into c283991331554e60a1f536235cf2d7b4(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:10,777 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:10,777 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/B, priority=13, startTime=1731712570299; duration=0sec 2024-11-15T23:16:10,777 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:10,777 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:B 2024-11-15T23:16:10,777 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:16:10,780 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:16:10,780 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/C is initiating minor compaction (all files) 2024-11-15T23:16:10,780 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/C in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:10,780 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/a84f9cd8eacc448dbb8ed3537419b652 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/a84f9cd8eacc448dbb8ed3537419b652 2024-11-15T23:16:10,780 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/1eebfe359b884d6493535beafe9cede7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/cad29a4b50ea44c7a6e377aeb67beb66, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/1c7705176b9a43cbae955c75696bcaad, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/aabc507ac2744c93bd6e1560a606e775] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=47.0 K 2024-11-15T23:16:10,782 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eebfe359b884d6493535beafe9cede7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731712567335 2024-11-15T23:16:10,783 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting cad29a4b50ea44c7a6e377aeb67beb66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731712567981 2024-11-15T23:16:10,784 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c7705176b9a43cbae955c75696bcaad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731712568685 2024-11-15T23:16:10,785 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting aabc507ac2744c93bd6e1560a606e775, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731712569856 2024-11-15T23:16:10,788 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/A of 083d01f7dba258590a0a4db0eb54177d into a84f9cd8eacc448dbb8ed3537419b652(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:10,788 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:10,788 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/A, priority=13, startTime=1731712570298; duration=0sec 2024-11-15T23:16:10,788 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:10,788 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:A 2024-11-15T23:16:10,797 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#C#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:10,798 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/cbf4d52997f1468aaa5af970efdb9788 is 50, key is test_row_0/C:col10/1731712569857/Put/seqid=0 2024-11-15T23:16:10,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742083_1259 (size=12241) 2024-11-15T23:16:10,836 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/cbf4d52997f1468aaa5af970efdb9788 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/cbf4d52997f1468aaa5af970efdb9788 2024-11-15T23:16:10,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712630835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712630836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,843 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/C of 083d01f7dba258590a0a4db0eb54177d into cbf4d52997f1468aaa5af970efdb9788(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:10,843 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:10,843 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/C, priority=12, startTime=1731712570299; duration=0sec 2024-11-15T23:16:10,843 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:10,843 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:C 2024-11-15T23:16:10,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:10,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:10,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:10,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:10,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:10,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:10,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T23:16:10,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/8762d1a9527347ae9b8e9cd44da25cbf 2024-11-15T23:16:10,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/4005db4634834d738849ec2ed5c5fefa is 50, key is test_row_1/B:col10/1731712570492/Put/seqid=0 2024-11-15T23:16:10,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742084_1260 (size=9657) 2024-11-15T23:16:10,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712630925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712630926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712630926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:11,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:11,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:11,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:11,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:11,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712631145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:11,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712631146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:11,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:11,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:11,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:11,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:11,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:11,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:11,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T23:16:11,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712631236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:11,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712631236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:11,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:11,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712631236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:11,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:11,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:11,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/4005db4634834d738849ec2ed5c5fefa 2024-11-15T23:16:11,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:11,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:11,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:11,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:11,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/715fb49d25cc4fe794407814fed375b1 is 50, key is test_row_1/C:col10/1731712570492/Put/seqid=0 2024-11-15T23:16:11,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742085_1261 (size=9657) 2024-11-15T23:16:11,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:11,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:11,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:11,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:11,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:11,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:11,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:11,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:11,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:11,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712631655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:11,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:11,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712631655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:11,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:11,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T23:16:11,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/715fb49d25cc4fe794407814fed375b1 2024-11-15T23:16:11,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/8762d1a9527347ae9b8e9cd44da25cbf as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/8762d1a9527347ae9b8e9cd44da25cbf 2024-11-15T23:16:11,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/8762d1a9527347ae9b8e9cd44da25cbf, entries=150, sequenceid=112, filesize=11.7 K 2024-11-15T23:16:11,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/4005db4634834d738849ec2ed5c5fefa as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/4005db4634834d738849ec2ed5c5fefa 2024-11-15T23:16:11,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/4005db4634834d738849ec2ed5c5fefa, entries=100, sequenceid=112, filesize=9.4 K 2024-11-15T23:16:11,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/715fb49d25cc4fe794407814fed375b1 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/715fb49d25cc4fe794407814fed375b1 2024-11-15T23:16:11,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/715fb49d25cc4fe794407814fed375b1, entries=100, sequenceid=112, filesize=9.4 K 2024-11-15T23:16:11,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 083d01f7dba258590a0a4db0eb54177d in 1250ms, sequenceid=112, compaction requested=true 2024-11-15T23:16:11,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:11,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:16:11,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:11,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:16:11,747 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:11,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:11,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:16:11,747 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:11,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:11,748 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36205 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:11,748 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:11,748 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/B is initiating minor compaction (all files) 2024-11-15T23:16:11,748 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/A is initiating minor compaction (all files) 2024-11-15T23:16:11,748 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/B in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:11,748 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/A in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:11,748 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/c283991331554e60a1f536235cf2d7b4, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2a7e4d42fba640dcaa7e8a6654be36a2, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/4005db4634834d738849ec2ed5c5fefa] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=33.1 K 2024-11-15T23:16:11,748 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/a84f9cd8eacc448dbb8ed3537419b652, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/52030626c2414c319b99a70f8b5dfb5e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/8762d1a9527347ae9b8e9cd44da25cbf] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=35.4 K 2024-11-15T23:16:11,748 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting c283991331554e60a1f536235cf2d7b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731712568685 2024-11-15T23:16:11,748 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting a84f9cd8eacc448dbb8ed3537419b652, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731712568685 2024-11-15T23:16:11,748 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a7e4d42fba640dcaa7e8a6654be36a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731712569856 2024-11-15T23:16:11,748 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52030626c2414c319b99a70f8b5dfb5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731712569856 2024-11-15T23:16:11,749 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 4005db4634834d738849ec2ed5c5fefa, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731712570492 2024-11-15T23:16:11,749 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8762d1a9527347ae9b8e9cd44da25cbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731712570492 2024-11-15T23:16:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:11,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=147.60 KB heapSize=425.53 KB 2024-11-15T23:16:11,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:11,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T23:16:11,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:11,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T23:16:11,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:11,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T23:16:11,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/a893be96577b48fbbb47854d596a610e is 50, key is test_row_0/A:col10/1731712571750/Put/seqid=0 2024-11-15T23:16:11,762 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#B#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:11,762 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/da6b53618bcc44fbb302b48a1c276957 is 50, key is test_row_0/B:col10/1731712569857/Put/seqid=0 2024-11-15T23:16:11,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:11,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712631763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:11,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:11,770 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#A#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:11,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:11,771 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/dfbf09fa25c140abb27546ec137da749 is 50, key is test_row_0/A:col10/1731712569857/Put/seqid=0 2024-11-15T23:16:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712631764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:11,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712631767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:11,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:11,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:11,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:11,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:11,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742086_1262 (size=12151) 2024-11-15T23:16:11,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742087_1263 (size=12309) 2024-11-15T23:16:11,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742088_1264 (size=12309) 2024-11-15T23:16:11,800 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/da6b53618bcc44fbb302b48a1c276957 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/da6b53618bcc44fbb302b48a1c276957 2024-11-15T23:16:11,805 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/B of 083d01f7dba258590a0a4db0eb54177d into da6b53618bcc44fbb302b48a1c276957(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:11,805 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:11,805 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/B, priority=13, startTime=1731712571747; duration=0sec 2024-11-15T23:16:11,805 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:11,805 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:B 2024-11-15T23:16:11,806 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T23:16:11,806 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T23:16:11,806 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T23:16:11,807 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. because compaction request was cancelled 2024-11-15T23:16:11,807 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:C 2024-11-15T23:16:11,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712631875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:11,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712631875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:11,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712631875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:11,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:11,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:11,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:11,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:11,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:11,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:12,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:12,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:12,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712632085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:12,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:12,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:12,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:12,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712632085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712632085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:12,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/a893be96577b48fbbb47854d596a610e 2024-11-15T23:16:12,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/f91f1e203d4e432db0983dca5b960881 is 50, key is test_row_0/B:col10/1731712571750/Put/seqid=0 2024-11-15T23:16:12,198 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/dfbf09fa25c140abb27546ec137da749 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/dfbf09fa25c140abb27546ec137da749 2024-11-15T23:16:12,203 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/A of 083d01f7dba258590a0a4db0eb54177d into dfbf09fa25c140abb27546ec137da749(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:12,203 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:12,203 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/A, priority=13, startTime=1731712571747; duration=0sec 2024-11-15T23:16:12,204 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:12,204 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:A 2024-11-15T23:16:12,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742089_1265 (size=12151) 2024-11-15T23:16:12,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/f91f1e203d4e432db0983dca5b960881 2024-11-15T23:16:12,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/73cb82066bf6407abf4047ae7f9ba8cd is 50, key is test_row_0/C:col10/1731712571750/Put/seqid=0 2024-11-15T23:16:12,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742090_1266 (size=12151) 2024-11-15T23:16:12,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:12,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:12,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:12,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:12,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:12,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:12,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:12,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:12,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:12,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:12,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:12,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:12,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712632396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:12,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712632396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:12,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712632396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:12,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:12,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:12,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:12,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:12,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:12,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:12,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/73cb82066bf6407abf4047ae7f9ba8cd 2024-11-15T23:16:12,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/a893be96577b48fbbb47854d596a610e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/a893be96577b48fbbb47854d596a610e 2024-11-15T23:16:12,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/a893be96577b48fbbb47854d596a610e, entries=150, sequenceid=154, filesize=11.9 K 2024-11-15T23:16:12,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/f91f1e203d4e432db0983dca5b960881 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/f91f1e203d4e432db0983dca5b960881 2024-11-15T23:16:12,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/f91f1e203d4e432db0983dca5b960881, entries=150, sequenceid=154, filesize=11.9 K 2024-11-15T23:16:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/73cb82066bf6407abf4047ae7f9ba8cd as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/73cb82066bf6407abf4047ae7f9ba8cd 2024-11-15T23:16:12,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/73cb82066bf6407abf4047ae7f9ba8cd, entries=150, sequenceid=154, filesize=11.9 K 2024-11-15T23:16:12,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~425.48 KB/435696, currentSize=33.54 KB/34350 for 083d01f7dba258590a0a4db0eb54177d in 886ms, sequenceid=154, compaction requested=true 2024-11-15T23:16:12,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:12,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:16:12,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:12,637 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T23:16:12,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:16:12,637 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T23:16:12,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:12,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:16:12,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:12,638 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T23:16:12,638 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T23:16:12,638 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. because compaction request was cancelled 2024-11-15T23:16:12,638 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:B 2024-11-15T23:16:12,638 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:12,638 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T23:16:12,638 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T23:16:12,638 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. because compaction request was cancelled 2024-11-15T23:16:12,638 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:A 2024-11-15T23:16:12,639 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:12,639 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/C is initiating minor compaction (all files) 2024-11-15T23:16:12,639 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/C in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:12,639 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/cbf4d52997f1468aaa5af970efdb9788, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/715fb49d25cc4fe794407814fed375b1, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/73cb82066bf6407abf4047ae7f9ba8cd] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=33.3 K 2024-11-15T23:16:12,640 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting cbf4d52997f1468aaa5af970efdb9788, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731712569856 2024-11-15T23:16:12,640 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 715fb49d25cc4fe794407814fed375b1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731712570492 2024-11-15T23:16:12,640 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 73cb82066bf6407abf4047ae7f9ba8cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731712570594 2024-11-15T23:16:12,646 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#C#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:12,647 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/62080690adad45059c8b8aa5aeee373f is 50, key is test_row_0/C:col10/1731712571750/Put/seqid=0 2024-11-15T23:16:12,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742091_1267 (size=12493) 2024-11-15T23:16:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:12,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T23:16:12,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:12,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:12,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:12,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:12,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:12,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:12,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/79cd09324bf4405ba2b5f09d2e7239e4 is 50, key is test_row_0/A:col10/1731712571762/Put/seqid=0 2024-11-15T23:16:12,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742092_1268 (size=14541) 2024-11-15T23:16:12,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:12,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:12,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:12,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:12,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:12,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T23:16:12,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712572699, max timestamp=1731712572708Num uniques 859; 2024-11-15T23:16:12,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712572699, max timestamp=1731712572708Num uniques 859; 2024-11-15T23:16:12,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:16:12,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712572699, max timestamp=1731712572708Num uniques 859; 2024-11-15T23:16:12,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:16:12,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:16:12,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712632774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:12,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712632775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:12,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:12,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:12,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:12,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:12,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:12,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:12,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:12,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:12,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712632886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:12,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712632886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:12,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:12,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712632905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:12,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712632905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:12,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712632905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:13,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:13,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:13,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:13,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:13,058 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/62080690adad45059c8b8aa5aeee373f as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/62080690adad45059c8b8aa5aeee373f 2024-11-15T23:16:13,067 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/C of 083d01f7dba258590a0a4db0eb54177d into 62080690adad45059c8b8aa5aeee373f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:13,067 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:13,067 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/C, priority=13, startTime=1731712572637; duration=0sec 2024-11-15T23:16:13,067 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:13,067 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:C 2024-11-15T23:16:13,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:13,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712633095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:13,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:13,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712633095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:13,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/79cd09324bf4405ba2b5f09d2e7239e4 2024-11-15T23:16:13,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/fea71a7b560349469e1c447184c3a003 is 50, key is test_row_0/B:col10/1731712571762/Put/seqid=0 2024-11-15T23:16:13,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742093_1269 (size=12151) 2024-11-15T23:16:13,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:13,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:13,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:13,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:13,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:13,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:13,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:13,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:13,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:13,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:13,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:13,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:13,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:13,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:13,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712633407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:13,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712633407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:13,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:13,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:13,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:13,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:13,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:13,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:13,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:13,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:13,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:13,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:13,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:13,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:13,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/fea71a7b560349469e1c447184c3a003 2024-11-15T23:16:13,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/0e5f3681d4fc445fb7d922961bc2133e is 50, key is test_row_0/C:col10/1731712571762/Put/seqid=0 2024-11-15T23:16:13,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742094_1270 (size=12151) 2024-11-15T23:16:13,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/0e5f3681d4fc445fb7d922961bc2133e 2024-11-15T23:16:13,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/79cd09324bf4405ba2b5f09d2e7239e4 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/79cd09324bf4405ba2b5f09d2e7239e4 2024-11-15T23:16:13,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/79cd09324bf4405ba2b5f09d2e7239e4, entries=200, sequenceid=166, filesize=14.2 K 2024-11-15T23:16:13,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/fea71a7b560349469e1c447184c3a003 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/fea71a7b560349469e1c447184c3a003 2024-11-15T23:16:13,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/fea71a7b560349469e1c447184c3a003, entries=150, sequenceid=166, filesize=11.9 K 2024-11-15T23:16:13,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/0e5f3681d4fc445fb7d922961bc2133e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/0e5f3681d4fc445fb7d922961bc2133e 2024-11-15T23:16:13,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/0e5f3681d4fc445fb7d922961bc2133e, entries=150, sequenceid=166, filesize=11.9 K 2024-11-15T23:16:13,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 083d01f7dba258590a0a4db0eb54177d in 950ms, sequenceid=166, compaction requested=true 2024-11-15T23:16:13,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:13,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:16:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:13,618 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:13,618 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:13,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:16:13,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:13,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:16:13,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:13,625 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:13,625 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:13,625 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/A is initiating minor compaction (all files) 2024-11-15T23:16:13,625 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/B is initiating minor compaction (all files) 2024-11-15T23:16:13,625 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/A in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:13,625 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/B in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:13,625 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/da6b53618bcc44fbb302b48a1c276957, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/f91f1e203d4e432db0983dca5b960881, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/fea71a7b560349469e1c447184c3a003] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=35.8 K 2024-11-15T23:16:13,625 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/dfbf09fa25c140abb27546ec137da749, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/a893be96577b48fbbb47854d596a610e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/79cd09324bf4405ba2b5f09d2e7239e4] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=38.1 K 2024-11-15T23:16:13,626 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting da6b53618bcc44fbb302b48a1c276957, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731712569857 2024-11-15T23:16:13,626 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfbf09fa25c140abb27546ec137da749, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731712569857 2024-11-15T23:16:13,627 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting a893be96577b48fbbb47854d596a610e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731712570594 2024-11-15T23:16:13,627 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting f91f1e203d4e432db0983dca5b960881, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731712570594 2024-11-15T23:16:13,628 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting fea71a7b560349469e1c447184c3a003, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731712571762 2024-11-15T23:16:13,628 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79cd09324bf4405ba2b5f09d2e7239e4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731712571762 2024-11-15T23:16:13,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:16:13,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:13,632 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-15T23:16:13,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:13,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:13,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:13,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:13,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:13,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:13,643 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#B#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:13,645 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/6f3e277c5e0848c2b47850f595fdfb16 is 50, key is test_row_0/B:col10/1731712571762/Put/seqid=0 2024-11-15T23:16:13,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/d0b892a5594f40aabf1cd33286eb9505 is 50, key is test_row_0/A:col10/1731712572731/Put/seqid=0 2024-11-15T23:16:13,659 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#A#compaction#227 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:13,659 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/ae125be6dc8a4a13959b49d61c46eec1 is 50, key is test_row_0/A:col10/1731712571762/Put/seqid=0 2024-11-15T23:16:13,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742095_1271 (size=12561) 2024-11-15T23:16:13,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742096_1272 (size=12151) 2024-11-15T23:16:13,711 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/d0b892a5594f40aabf1cd33286eb9505 2024-11-15T23:16:13,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742097_1273 (size=12561) 2024-11-15T23:16:13,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/f3187a506d7a436eaf652883d4bf7654 is 50, key is test_row_0/B:col10/1731712572731/Put/seqid=0 2024-11-15T23:16:13,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742098_1274 (size=12151) 2024-11-15T23:16:13,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:13,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:13,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712633926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:13,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712633927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:13,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:13,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:13,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:13,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712633933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:13,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:13,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712633934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712633936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712634045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712634045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712634046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712634046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712634046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,103 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/6f3e277c5e0848c2b47850f595fdfb16 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/6f3e277c5e0848c2b47850f595fdfb16 2024-11-15T23:16:14,111 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/B of 083d01f7dba258590a0a4db0eb54177d into 6f3e277c5e0848c2b47850f595fdfb16(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:14,111 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:14,111 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/B, priority=13, startTime=1731712573618; duration=0sec 2024-11-15T23:16:14,111 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:14,111 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:B 2024-11-15T23:16:14,111 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T23:16:14,117 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T23:16:14,117 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T23:16:14,117 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. because compaction request was cancelled 2024-11-15T23:16:14,117 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:C 2024-11-15T23:16:14,133 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/ae125be6dc8a4a13959b49d61c46eec1 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/ae125be6dc8a4a13959b49d61c46eec1 2024-11-15T23:16:14,141 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/A of 083d01f7dba258590a0a4db0eb54177d into ae125be6dc8a4a13959b49d61c46eec1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:14,141 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:14,141 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/A, priority=13, startTime=1731712573617; duration=0sec 2024-11-15T23:16:14,141 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:14,141 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:A 2024-11-15T23:16:14,172 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/f3187a506d7a436eaf652883d4bf7654 2024-11-15T23:16:14,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/89d693e65d544a13a707cdf5e4563d86 is 50, key is test_row_0/C:col10/1731712572731/Put/seqid=0 2024-11-15T23:16:14,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742099_1275 (size=12151) 2024-11-15T23:16:14,240 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/89d693e65d544a13a707cdf5e4563d86 2024-11-15T23:16:14,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/d0b892a5594f40aabf1cd33286eb9505 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/d0b892a5594f40aabf1cd33286eb9505 2024-11-15T23:16:14,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712634255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712634255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,262 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/d0b892a5594f40aabf1cd33286eb9505, entries=150, sequenceid=203, filesize=11.9 K 2024-11-15T23:16:14,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712634255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/f3187a506d7a436eaf652883d4bf7654 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/f3187a506d7a436eaf652883d4bf7654 2024-11-15T23:16:14,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712634266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712634266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,278 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/f3187a506d7a436eaf652883d4bf7654, entries=150, sequenceid=203, filesize=11.9 K 2024-11-15T23:16:14,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/89d693e65d544a13a707cdf5e4563d86 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/89d693e65d544a13a707cdf5e4563d86 2024-11-15T23:16:14,295 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/89d693e65d544a13a707cdf5e4563d86, entries=150, sequenceid=203, filesize=11.9 K 2024-11-15T23:16:14,302 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 083d01f7dba258590a0a4db0eb54177d in 669ms, sequenceid=203, compaction requested=true 2024-11-15T23:16:14,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:14,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:14,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-15T23:16:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-15T23:16:14,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-15T23:16:14,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7560 sec 2024-11-15T23:16:14,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 3.7620 sec 2024-11-15T23:16:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:14,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T23:16:14,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:14,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:14,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:14,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:14,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:14,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:14,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/12e7b00348f24d0dba07281cd9cfccad is 50, key is test_row_0/A:col10/1731712573932/Put/seqid=0 2024-11-15T23:16:14,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742100_1276 (size=14541) 2024-11-15T23:16:14,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/12e7b00348f24d0dba07281cd9cfccad 2024-11-15T23:16:14,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/2b75b3cbbfba4af7b2cb6341e7f04ab8 is 50, key is test_row_0/B:col10/1731712573932/Put/seqid=0 2024-11-15T23:16:14,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731712574598, max timestamp=1731712574614Num uniques 859; 2024-11-15T23:16:14,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731712574598, max timestamp=1731712574614Num uniques 859; 2024-11-15T23:16:14,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T23:16:14,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T23:16:14,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731712574598, max timestamp=1731712574614Num uniques 859; 2024-11-15T23:16:14,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T23:16:14,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742101_1277 (size=12151) 2024-11-15T23:16:14,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/2b75b3cbbfba4af7b2cb6341e7f04ab8 2024-11-15T23:16:14,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/67a9ae22632e4534b5994ee7ff6c6b64 is 50, key is test_row_0/C:col10/1731712573932/Put/seqid=0 2024-11-15T23:16:14,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712634651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712634651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712634653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712634660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712634663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742102_1278 (size=12151) 2024-11-15T23:16:14,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/67a9ae22632e4534b5994ee7ff6c6b64 2024-11-15T23:16:14,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/12e7b00348f24d0dba07281cd9cfccad as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/12e7b00348f24d0dba07281cd9cfccad 2024-11-15T23:16:14,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/12e7b00348f24d0dba07281cd9cfccad, entries=200, sequenceid=216, filesize=14.2 K 2024-11-15T23:16:14,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/2b75b3cbbfba4af7b2cb6341e7f04ab8 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2b75b3cbbfba4af7b2cb6341e7f04ab8 2024-11-15T23:16:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T23:16:14,716 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:14,716 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:16:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-15T23:16:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T23:16:14,719 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:16:14,720 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:16:14,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:16:14,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2b75b3cbbfba4af7b2cb6341e7f04ab8, entries=150, sequenceid=216, filesize=11.9 K 2024-11-15T23:16:14,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/67a9ae22632e4534b5994ee7ff6c6b64 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/67a9ae22632e4534b5994ee7ff6c6b64 2024-11-15T23:16:14,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/67a9ae22632e4534b5994ee7ff6c6b64, entries=150, sequenceid=216, filesize=11.9 K 2024-11-15T23:16:14,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 083d01f7dba258590a0a4db0eb54177d in 174ms, sequenceid=216, compaction requested=true 2024-11-15T23:16:14,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:14,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:16:14,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:14,743 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:14,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:16:14,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:14,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:16:14,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T23:16:14,743 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:16:14,744 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:14,744 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/A is initiating minor compaction (all files) 2024-11-15T23:16:14,745 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/A in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:14,745 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/ae125be6dc8a4a13959b49d61c46eec1, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/d0b892a5594f40aabf1cd33286eb9505, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/12e7b00348f24d0dba07281cd9cfccad] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=38.3 K 2024-11-15T23:16:14,745 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae125be6dc8a4a13959b49d61c46eec1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731712571762 2024-11-15T23:16:14,745 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:16:14,746 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/C is initiating minor compaction (all files) 2024-11-15T23:16:14,746 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/C in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:14,746 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/62080690adad45059c8b8aa5aeee373f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/0e5f3681d4fc445fb7d922961bc2133e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/89d693e65d544a13a707cdf5e4563d86, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/67a9ae22632e4534b5994ee7ff6c6b64] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=47.8 K 2024-11-15T23:16:14,746 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 62080690adad45059c8b8aa5aeee373f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731712570594 2024-11-15T23:16:14,746 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0b892a5594f40aabf1cd33286eb9505, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731712572731 2024-11-15T23:16:14,746 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12e7b00348f24d0dba07281cd9cfccad, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731712573922 2024-11-15T23:16:14,747 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e5f3681d4fc445fb7d922961bc2133e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731712571762 2024-11-15T23:16:14,747 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 89d693e65d544a13a707cdf5e4563d86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731712572731 2024-11-15T23:16:14,748 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 67a9ae22632e4534b5994ee7ff6c6b64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731712573932 2024-11-15T23:16:14,766 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#A#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:14,767 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/91757c8c3b8e455f929e66fe448e4f16 is 50, key is test_row_0/A:col10/1731712573932/Put/seqid=0 2024-11-15T23:16:14,779 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#C#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:14,779 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/c324a6a7db274453ad3f4eae149e0de6 is 50, key is test_row_0/C:col10/1731712573932/Put/seqid=0 2024-11-15T23:16:14,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-15T23:16:14,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:14,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:14,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:14,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:14,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:14,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:14,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742104_1280 (size=12629) 2024-11-15T23:16:14,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/c30430ab01904d1894b88cea008730e7 is 50, key is test_row_0/A:col10/1731712574660/Put/seqid=0 2024-11-15T23:16:14,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742103_1279 (size=12663) 2024-11-15T23:16:14,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712634802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712634802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712634802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712634803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712634803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,819 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/91757c8c3b8e455f929e66fe448e4f16 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/91757c8c3b8e455f929e66fe448e4f16 2024-11-15T23:16:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T23:16:14,828 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/A of 083d01f7dba258590a0a4db0eb54177d into 91757c8c3b8e455f929e66fe448e4f16(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:14,828 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:14,828 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/A, priority=13, startTime=1731712574743; duration=0sec 2024-11-15T23:16:14,828 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:14,828 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:A 2024-11-15T23:16:14,828 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:14,833 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:14,833 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/B is initiating minor compaction (all files) 2024-11-15T23:16:14,833 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/B in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:14,833 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/6f3e277c5e0848c2b47850f595fdfb16, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/f3187a506d7a436eaf652883d4bf7654, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2b75b3cbbfba4af7b2cb6341e7f04ab8] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=36.0 K 2024-11-15T23:16:14,836 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f3e277c5e0848c2b47850f595fdfb16, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731712571762 2024-11-15T23:16:14,837 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3187a506d7a436eaf652883d4bf7654, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731712572731 2024-11-15T23:16:14,842 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b75b3cbbfba4af7b2cb6341e7f04ab8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731712573932 2024-11-15T23:16:14,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742105_1281 (size=12151) 2024-11-15T23:16:14,864 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#B#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:14,864 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/95fddf6a40824a688a2b7e65b80cef08 is 50, key is test_row_0/B:col10/1731712573932/Put/seqid=0 2024-11-15T23:16:14,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T23:16:14,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:14,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:14,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:14,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:14,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:14,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742106_1282 (size=12663) 2024-11-15T23:16:14,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712634915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712634915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712634915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712634916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:14,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712634925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:15,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T23:16:15,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:15,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:15,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:15,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T23:16:15,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712635135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712635136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712635136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712635136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:15,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712635136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:15,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:15,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T23:16:15,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:15,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:15,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:15,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,195 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/c324a6a7db274453ad3f4eae149e0de6 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/c324a6a7db274453ad3f4eae149e0de6 2024-11-15T23:16:15,200 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/C of 083d01f7dba258590a0a4db0eb54177d into c324a6a7db274453ad3f4eae149e0de6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:15,200 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:15,200 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/C, priority=12, startTime=1731712574743; duration=0sec 2024-11-15T23:16:15,200 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:15,200 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:C 2024-11-15T23:16:15,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/c30430ab01904d1894b88cea008730e7 2024-11-15T23:16:15,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/9549201f819c47e1a8c52563561124ab is 50, key is test_row_0/B:col10/1731712574660/Put/seqid=0 2024-11-15T23:16:15,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742107_1283 (size=12151) 2024-11-15T23:16:15,312 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/95fddf6a40824a688a2b7e65b80cef08 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/95fddf6a40824a688a2b7e65b80cef08 2024-11-15T23:16:15,317 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/B of 083d01f7dba258590a0a4db0eb54177d into 95fddf6a40824a688a2b7e65b80cef08(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:15,317 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:15,317 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/B, priority=13, startTime=1731712574743; duration=0sec 2024-11-15T23:16:15,317 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:15,317 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:B 2024-11-15T23:16:15,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T23:16:15,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:15,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:15,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:15,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T23:16:15,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712635445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:15,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712635445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712635445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:15,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712635445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:15,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712635445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:15,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:15,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T23:16:15,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:15,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:15,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:15,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T23:16:15,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:15,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:15,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:15,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/9549201f819c47e1a8c52563561124ab 2024-11-15T23:16:15,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/9673c8c260644db29f5973a03b1219a4 is 50, key is test_row_0/C:col10/1731712574660/Put/seqid=0 2024-11-15T23:16:15,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742108_1284 (size=12151) 2024-11-15T23:16:15,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T23:16:15,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:15,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:15,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:15,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T23:16:15,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T23:16:15,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:15,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:15,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:15,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:15,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712635956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:15,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712635956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:15,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712635956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712635956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712635956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:16,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/9673c8c260644db29f5973a03b1219a4 2024-11-15T23:16:16,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/c30430ab01904d1894b88cea008730e7 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/c30430ab01904d1894b88cea008730e7 2024-11-15T23:16:16,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/c30430ab01904d1894b88cea008730e7, entries=150, sequenceid=251, filesize=11.9 K 2024-11-15T23:16:16,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/9549201f819c47e1a8c52563561124ab as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/9549201f819c47e1a8c52563561124ab 2024-11-15T23:16:16,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/9549201f819c47e1a8c52563561124ab, entries=150, sequenceid=251, filesize=11.9 K 2024-11-15T23:16:16,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/9673c8c260644db29f5973a03b1219a4 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/9673c8c260644db29f5973a03b1219a4 2024-11-15T23:16:16,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/9673c8c260644db29f5973a03b1219a4, entries=150, sequenceid=251, filesize=11.9 K 2024-11-15T23:16:16,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=60.38 KB/61830 for 083d01f7dba258590a0a4db0eb54177d in 1305ms, sequenceid=251, compaction requested=false 2024-11-15T23:16:16,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:16,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T23:16:16,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:16,109 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T23:16:16,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:16,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:16,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:16,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:16,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:16,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:16,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/f1eab116062c46c49ad5b03724662114 is 50, key is test_row_0/A:col10/1731712574800/Put/seqid=0 2024-11-15T23:16:16,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742109_1285 (size=12251) 2024-11-15T23:16:16,517 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/f1eab116062c46c49ad5b03724662114 2024-11-15T23:16:16,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/4fbc1f23ac5d40e3afd08fb5aee27878 is 50, key is test_row_0/B:col10/1731712574800/Put/seqid=0 2024-11-15T23:16:16,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742110_1286 (size=12251) 2024-11-15T23:16:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T23:16:16,929 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/4fbc1f23ac5d40e3afd08fb5aee27878 2024-11-15T23:16:16,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/c9c7a10fff37447c84d1d0c1380a3a02 is 50, key is test_row_0/C:col10/1731712574800/Put/seqid=0 2024-11-15T23:16:16,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742111_1287 (size=12251) 2024-11-15T23:16:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:16,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:16,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712636984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:16,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712636985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:16,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712636986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:16,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712636987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:16,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712636989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:16,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:16,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:16,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712637095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712637095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712637095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712637095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712637095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712637305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712637305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712637305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712637305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712637305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,339 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/c9c7a10fff37447c84d1d0c1380a3a02 2024-11-15T23:16:17,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/f1eab116062c46c49ad5b03724662114 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/f1eab116062c46c49ad5b03724662114 2024-11-15T23:16:17,347 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/f1eab116062c46c49ad5b03724662114, entries=150, sequenceid=266, filesize=12.0 K 2024-11-15T23:16:17,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/4fbc1f23ac5d40e3afd08fb5aee27878 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/4fbc1f23ac5d40e3afd08fb5aee27878 2024-11-15T23:16:17,352 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/4fbc1f23ac5d40e3afd08fb5aee27878, entries=150, sequenceid=266, filesize=12.0 K 2024-11-15T23:16:17,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/c9c7a10fff37447c84d1d0c1380a3a02 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/c9c7a10fff37447c84d1d0c1380a3a02 2024-11-15T23:16:17,357 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/c9c7a10fff37447c84d1d0c1380a3a02, entries=150, sequenceid=266, filesize=12.0 K 2024-11-15T23:16:17,358 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 083d01f7dba258590a0a4db0eb54177d in 1249ms, sequenceid=266, compaction requested=true 2024-11-15T23:16:17,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:17,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:17,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-15T23:16:17,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-15T23:16:17,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-15T23:16:17,361 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6390 sec 2024-11-15T23:16:17,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.6450 sec 2024-11-15T23:16:17,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:17,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-15T23:16:17,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712576984, max timestamp=1731712577617Num uniques 859; 2024-11-15T23:16:17,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712576984, max timestamp=1731712577617Num uniques 859; 2024-11-15T23:16:17,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:16:17,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:16:17,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712576984, max timestamp=1731712577617Num uniques 859; 2024-11-15T23:16:17,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:16:17,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:17,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:17,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:17,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:17,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:17,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:17,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/6d5e707df86e48eaa6be4e41351cb92f is 50, key is test_row_0/A:col10/1731712576988/Put/seqid=0 2024-11-15T23:16:17,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742112_1288 (size=12301) 2024-11-15T23:16:17,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731712577637, max timestamp=1731712577646Num uniques 859; 2024-11-15T23:16:17,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731712577637, max timestamp=1731712577646Num uniques 859; 2024-11-15T23:16:17,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T23:16:17,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T23:16:17,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731712577637, max timestamp=1731712577646Num uniques 859; 2024-11-15T23:16:17,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T23:16:17,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712637691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712637692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712637693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712637694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712637695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712637805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712637805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712637805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712637805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712637805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:18,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712638015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:18,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712638015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:18,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712638015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:18,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712638015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:18,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712638015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:18,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:18,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/6d5e707df86e48eaa6be4e41351cb92f 2024-11-15T23:16:18,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/aa9d39db2ea349759797d0315868488d is 50, key is test_row_0/B:col10/1731712576988/Put/seqid=0 2024-11-15T23:16:18,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742113_1289 (size=12301) 2024-11-15T23:16:18,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/aa9d39db2ea349759797d0315868488d 2024-11-15T23:16:18,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/2645b6aa1a5949c89129b1ad74f0b50f is 50, key is test_row_0/C:col10/1731712576988/Put/seqid=0 2024-11-15T23:16:18,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742114_1290 (size=12301) 2024-11-15T23:16:18,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712638325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:18,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:18,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712638325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:18,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:18,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:18,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712638325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:18,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712638325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:18,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712638325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:18,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/2645b6aa1a5949c89129b1ad74f0b50f 2024-11-15T23:16:18,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/6d5e707df86e48eaa6be4e41351cb92f as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/6d5e707df86e48eaa6be4e41351cb92f 2024-11-15T23:16:18,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/6d5e707df86e48eaa6be4e41351cb92f, entries=150, sequenceid=288, filesize=12.0 K 2024-11-15T23:16:18,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/aa9d39db2ea349759797d0315868488d as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/aa9d39db2ea349759797d0315868488d 2024-11-15T23:16:18,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/aa9d39db2ea349759797d0315868488d, entries=150, sequenceid=288, filesize=12.0 K 2024-11-15T23:16:18,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/2645b6aa1a5949c89129b1ad74f0b50f as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/2645b6aa1a5949c89129b1ad74f0b50f 2024-11-15T23:16:18,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/2645b6aa1a5949c89129b1ad74f0b50f, entries=150, sequenceid=288, filesize=12.0 K 2024-11-15T23:16:18,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for 083d01f7dba258590a0a4db0eb54177d in 902ms, sequenceid=288, compaction requested=true 2024-11-15T23:16:18,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:18,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:16:18,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:18,520 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:16:18,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:16:18,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:18,520 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:16:18,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:16:18,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:18,527 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49366 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:16:18,527 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/A is initiating minor compaction (all files) 2024-11-15T23:16:18,527 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49366 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:16:18,527 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/B is initiating minor compaction (all files) 2024-11-15T23:16:18,527 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/A in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:18,527 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/B in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:18,527 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/91757c8c3b8e455f929e66fe448e4f16, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/c30430ab01904d1894b88cea008730e7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/f1eab116062c46c49ad5b03724662114, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/6d5e707df86e48eaa6be4e41351cb92f] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=48.2 K 2024-11-15T23:16:18,527 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/95fddf6a40824a688a2b7e65b80cef08, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/9549201f819c47e1a8c52563561124ab, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/4fbc1f23ac5d40e3afd08fb5aee27878, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/aa9d39db2ea349759797d0315868488d] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=48.2 K 2024-11-15T23:16:18,528 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 95fddf6a40824a688a2b7e65b80cef08, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731712573932 2024-11-15T23:16:18,528 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91757c8c3b8e455f929e66fe448e4f16, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731712573932 2024-11-15T23:16:18,528 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting c30430ab01904d1894b88cea008730e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731712574644 2024-11-15T23:16:18,528 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 9549201f819c47e1a8c52563561124ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731712574644 2024-11-15T23:16:18,528 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1eab116062c46c49ad5b03724662114, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731712574785 2024-11-15T23:16:18,529 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fbc1f23ac5d40e3afd08fb5aee27878, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731712574785 2024-11-15T23:16:18,529 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting aa9d39db2ea349759797d0315868488d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731712576986 2024-11-15T23:16:18,529 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d5e707df86e48eaa6be4e41351cb92f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731712576986 2024-11-15T23:16:18,539 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#A#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:18,539 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/72947a430de746cdbd4c7a1241947b44 is 50, key is test_row_0/A:col10/1731712576988/Put/seqid=0 2024-11-15T23:16:18,540 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#B#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:18,541 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/9bb3c62e00f84ffc8a8301be952c1b88 is 50, key is test_row_0/B:col10/1731712576988/Put/seqid=0 2024-11-15T23:16:18,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742116_1292 (size=12949) 2024-11-15T23:16:18,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742115_1291 (size=12949) 2024-11-15T23:16:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:18,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=161.02 KB heapSize=451.31 KB 2024-11-15T23:16:18,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:18,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T23:16:18,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:18,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T23:16:18,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:18,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T23:16:18,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/939cbbc0cbd9446c8f115deb0204d116 is 50, key is test_row_0/A:col10/1731712578839/Put/seqid=0 2024-11-15T23:16:18,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712638843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:18,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:18,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712638846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:18,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742117_1293 (size=14741) 2024-11-15T23:16:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:18,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712638846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:18,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712638847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:18,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:18,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712638849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T23:16:18,875 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:18,876 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:16:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-15T23:16:18,879 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:16:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T23:16:18,881 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:16:18,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:16:18,949 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/72947a430de746cdbd4c7a1241947b44 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/72947a430de746cdbd4c7a1241947b44 2024-11-15T23:16:18,949 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/9bb3c62e00f84ffc8a8301be952c1b88 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/9bb3c62e00f84ffc8a8301be952c1b88 2024-11-15T23:16:18,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712638962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:18,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712638962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:18,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:18,967 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/A of 083d01f7dba258590a0a4db0eb54177d into 72947a430de746cdbd4c7a1241947b44(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:18,967 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:18,967 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/A, priority=12, startTime=1731712578520; duration=0sec 2024-11-15T23:16:18,967 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/B of 083d01f7dba258590a0a4db0eb54177d into 9bb3c62e00f84ffc8a8301be952c1b88(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:18,967 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:18,967 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:18,967 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:A 2024-11-15T23:16:18,967 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/B, priority=12, startTime=1731712578520; duration=0sec 2024-11-15T23:16:18,967 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:16:18,967 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:18,967 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:B 2024-11-15T23:16:18,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712638965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:18,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:18,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:18,970 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49332 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:16:18,970 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/C is initiating minor compaction (all files) 2024-11-15T23:16:18,970 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/C in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:18,970 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/c324a6a7db274453ad3f4eae149e0de6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/9673c8c260644db29f5973a03b1219a4, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/c9c7a10fff37447c84d1d0c1380a3a02, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/2645b6aa1a5949c89129b1ad74f0b50f] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=48.2 K 2024-11-15T23:16:18,970 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting c324a6a7db274453ad3f4eae149e0de6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731712573932 2024-11-15T23:16:18,972 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9673c8c260644db29f5973a03b1219a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731712574644 2024-11-15T23:16:18,972 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9c7a10fff37447c84d1d0c1380a3a02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731712574785 2024-11-15T23:16:18,972 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2645b6aa1a5949c89129b1ad74f0b50f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731712576986 2024-11-15T23:16:18,985 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#C#compaction#248 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T23:16:18,986 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/571b1de536324b3ebaa70e2f20551ae0 is 50, key is test_row_0/C:col10/1731712576988/Put/seqid=0 2024-11-15T23:16:18,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742118_1294 (size=12915) 2024-11-15T23:16:19,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T23:16:19,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:19,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:19,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712639175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:19,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:19,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712639175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:19,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:19,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712639175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:19,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:19,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:19,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T23:16:19,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:19,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:19,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:19,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T23:16:19,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/939cbbc0cbd9446c8f115deb0204d116 2024-11-15T23:16:19,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/79fda4c3b99f4fe58dd2583a60d3ee32 is 50, key is test_row_0/B:col10/1731712578839/Put/seqid=0 2024-11-15T23:16:19,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742119_1295 (size=12301) 2024-11-15T23:16:19,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T23:16:19,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:19,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:19,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:19,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,400 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/571b1de536324b3ebaa70e2f20551ae0 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/571b1de536324b3ebaa70e2f20551ae0 2024-11-15T23:16:19,407 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/C of 083d01f7dba258590a0a4db0eb54177d into 571b1de536324b3ebaa70e2f20551ae0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:19,407 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:19,407 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/C, priority=12, startTime=1731712578520; duration=0sec 2024-11-15T23:16:19,407 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:19,407 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:C 2024-11-15T23:16:19,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:19,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712639485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:19,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:19,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712639485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712639485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:19,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T23:16:19,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:19,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:19,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:19,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T23:16:19,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T23:16:19,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:19,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:19,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:19,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/79fda4c3b99f4fe58dd2583a60d3ee32 2024-11-15T23:16:19,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/602fa765ff724d4582e351df24ac5c14 is 50, key is test_row_0/C:col10/1731712578839/Put/seqid=0 2024-11-15T23:16:19,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742120_1296 (size=12301) 2024-11-15T23:16:19,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T23:16:19,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:19,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:19,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:19,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:19,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712639865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:19,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:19,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712639865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:19,868 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in 083d01f7dba258590a0a4db0eb54177d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:16:19,868 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_0' in 083d01f7dba258590a0a4db0eb54177d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:16:19,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:19,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:19,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T23:16:19,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:19,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:19,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:19,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:19,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712639995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:19,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:19,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712639995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:19,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:19,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:19,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:19,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712639995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:20,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T23:16:20,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/602fa765ff724d4582e351df24ac5c14 2024-11-15T23:16:20,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/939cbbc0cbd9446c8f115deb0204d116 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/939cbbc0cbd9446c8f115deb0204d116 2024-11-15T23:16:20,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/939cbbc0cbd9446c8f115deb0204d116, entries=200, sequenceid=328, filesize=14.4 K 2024-11-15T23:16:20,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/79fda4c3b99f4fe58dd2583a60d3ee32 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/79fda4c3b99f4fe58dd2583a60d3ee32 2024-11-15T23:16:20,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/79fda4c3b99f4fe58dd2583a60d3ee32, entries=150, sequenceid=328, filesize=12.0 K 2024-11-15T23:16:20,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/602fa765ff724d4582e351df24ac5c14 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/602fa765ff724d4582e351df24ac5c14 2024-11-15T23:16:20,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/602fa765ff724d4582e351df24ac5c14, entries=150, sequenceid=328, filesize=12.0 K 2024-11-15T23:16:20,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~491.11 KB/502896, currentSize=13.42 KB/13740 for 083d01f7dba258590a0a4db0eb54177d in 1250ms, sequenceid=328, compaction requested=false 2024-11-15T23:16:20,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:20,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T23:16:20,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:20,105 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-15T23:16:20,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:20,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:20,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:20,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:20,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:20,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:20,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/61e3ba7a90944078bf3e6302e329f944 is 50, key is test_row_2/A:col10/1731712578847/Put/seqid=0 2024-11-15T23:16:20,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742121_1297 (size=7415) 2024-11-15T23:16:20,513 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/61e3ba7a90944078bf3e6302e329f944 2024-11-15T23:16:20,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/02c81d48fc5441cb8f655ef86fd4b19c is 50, key is test_row_2/B:col10/1731712578847/Put/seqid=0 2024-11-15T23:16:20,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742122_1298 (size=7415) 2024-11-15T23:16:20,528 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/02c81d48fc5441cb8f655ef86fd4b19c 2024-11-15T23:16:20,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/8937567aa8344302aa4d359e50dff17f is 50, key is test_row_2/C:col10/1731712578847/Put/seqid=0 2024-11-15T23:16:20,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742123_1299 (size=7415) 2024-11-15T23:16:20,942 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/8937567aa8344302aa4d359e50dff17f 2024-11-15T23:16:20,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/61e3ba7a90944078bf3e6302e329f944 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/61e3ba7a90944078bf3e6302e329f944 2024-11-15T23:16:20,960 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/61e3ba7a90944078bf3e6302e329f944, entries=50, sequenceid=336, filesize=7.2 K 2024-11-15T23:16:20,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/02c81d48fc5441cb8f655ef86fd4b19c as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/02c81d48fc5441cb8f655ef86fd4b19c 2024-11-15T23:16:20,970 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/02c81d48fc5441cb8f655ef86fd4b19c, entries=50, sequenceid=336, filesize=7.2 K 2024-11-15T23:16:20,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/8937567aa8344302aa4d359e50dff17f as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/8937567aa8344302aa4d359e50dff17f 2024-11-15T23:16:20,977 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/8937567aa8344302aa4d359e50dff17f, entries=50, sequenceid=336, filesize=7.2 K 2024-11-15T23:16:20,978 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 083d01f7dba258590a0a4db0eb54177d in 873ms, sequenceid=336, compaction requested=true 2024-11-15T23:16:20,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:20,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:20,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-15T23:16:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-15T23:16:20,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-15T23:16:20,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1000 sec 2024-11-15T23:16:20,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.1090 sec 2024-11-15T23:16:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T23:16:21,026 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:21,027 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:16:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-15T23:16:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T23:16:21,029 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:16:21,030 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:16:21,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:16:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:21,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T23:16:21,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:21,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:21,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:21,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:21,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:21,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:21,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/67c3d9eaced94158a8987a18fd36a9b8 is 50, key is test_row_0/A:col10/1731712581029/Put/seqid=0 2024-11-15T23:16:21,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712581057, max timestamp=1731712581073Num uniques 859; 2024-11-15T23:16:21,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712581057, max timestamp=1731712581073Num uniques 859; 2024-11-15T23:16:21,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:16:21,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:16:21,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742124_1300 (size=14741) 2024-11-15T23:16:21,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712581057, max timestamp=1731712581073Num uniques 859; 2024-11-15T23:16:21,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:16:21,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712641109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:21,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712641111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:21,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:21,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712641115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T23:16:21,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T23:16:21,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:21,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:21,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:21,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:21,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:21,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712641225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:21,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712641225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:21,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712641226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:21,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T23:16:21,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:21,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:21,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:21,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:21,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T23:16:21,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712641435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:21,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712641435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:21,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712641435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:21,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/67c3d9eaced94158a8987a18fd36a9b8 2024-11-15T23:16:21,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/25d6ef218fdd47139319c467e575cf32 is 50, key is test_row_0/B:col10/1731712581029/Put/seqid=0 2024-11-15T23:16:21,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T23:16:21,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:21,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:21,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:21,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:21,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:21,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:21,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742125_1301 (size=12301) 2024-11-15T23:16:21,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T23:16:21,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:21,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:21,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:21,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:21,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T23:16:21,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712641745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:21,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712641746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:21,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712641756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:21,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:21,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T23:16:21,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:21,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:21,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:21,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:21,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:21,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712641885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:21,890 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in 083d01f7dba258590a0a4db0eb54177d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:16:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:21,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:21,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712641896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:21,900 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_0' in 083d01f7dba258590a0a4db0eb54177d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:16:21,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:21,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:21,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/25d6ef218fdd47139319c467e575cf32 2024-11-15T23:16:21,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/f5afcf9f736b465cae6f937363da4201 is 50, key is test_row_0/C:col10/1731712581029/Put/seqid=0 2024-11-15T23:16:21,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T23:16:21,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:21,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:21,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:21,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:21,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:21,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742126_1302 (size=12301) 2024-11-15T23:16:21,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/f5afcf9f736b465cae6f937363da4201 2024-11-15T23:16:21,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/67c3d9eaced94158a8987a18fd36a9b8 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/67c3d9eaced94158a8987a18fd36a9b8 2024-11-15T23:16:21,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/67c3d9eaced94158a8987a18fd36a9b8, entries=200, sequenceid=347, filesize=14.4 K 2024-11-15T23:16:21,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/25d6ef218fdd47139319c467e575cf32 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/25d6ef218fdd47139319c467e575cf32 2024-11-15T23:16:21,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/25d6ef218fdd47139319c467e575cf32, entries=150, sequenceid=347, filesize=12.0 K 2024-11-15T23:16:21,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/f5afcf9f736b465cae6f937363da4201 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/f5afcf9f736b465cae6f937363da4201 2024-11-15T23:16:21,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/f5afcf9f736b465cae6f937363da4201, entries=150, sequenceid=347, filesize=12.0 K 2024-11-15T23:16:21,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 083d01f7dba258590a0a4db0eb54177d in 967ms, sequenceid=347, compaction requested=true 2024-11-15T23:16:21,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:21,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:16:21,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:21,998 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:16:21,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:16:21,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:21,998 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:16:21,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:16:21,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:22,000 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44966 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:16:22,000 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/B is initiating minor compaction (all files) 2024-11-15T23:16:22,000 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/B in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:22,000 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/9bb3c62e00f84ffc8a8301be952c1b88, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/79fda4c3b99f4fe58dd2583a60d3ee32, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/02c81d48fc5441cb8f655ef86fd4b19c, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/25d6ef218fdd47139319c467e575cf32] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=43.9 K 2024-11-15T23:16:22,001 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bb3c62e00f84ffc8a8301be952c1b88, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731712576986 2024-11-15T23:16:22,001 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49846 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:16:22,001 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/A is initiating minor compaction (all files) 2024-11-15T23:16:22,001 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/A in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:22,001 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/72947a430de746cdbd4c7a1241947b44, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/939cbbc0cbd9446c8f115deb0204d116, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/61e3ba7a90944078bf3e6302e329f944, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/67c3d9eaced94158a8987a18fd36a9b8] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=48.7 K 2024-11-15T23:16:22,004 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72947a430de746cdbd4c7a1241947b44, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731712576986 2024-11-15T23:16:22,004 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 79fda4c3b99f4fe58dd2583a60d3ee32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731712577691 2024-11-15T23:16:22,004 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 02c81d48fc5441cb8f655ef86fd4b19c, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731712578847 2024-11-15T23:16:22,004 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 939cbbc0cbd9446c8f115deb0204d116, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731712577691 2024-11-15T23:16:22,005 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 25d6ef218fdd47139319c467e575cf32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731712581023 2024-11-15T23:16:22,005 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61e3ba7a90944078bf3e6302e329f944, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731712578847 2024-11-15T23:16:22,005 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67c3d9eaced94158a8987a18fd36a9b8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731712581023 2024-11-15T23:16:22,027 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#B#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:22,027 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/fb4910cbde6343ffa4c7cb1c3a6fbba3 is 50, key is test_row_0/B:col10/1731712581029/Put/seqid=0 2024-11-15T23:16:22,031 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#A#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:22,032 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/19a0b2c5d3824dd6b1b818b02e47490e is 50, key is test_row_0/A:col10/1731712581029/Put/seqid=0 2024-11-15T23:16:22,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742127_1303 (size=13085) 2024-11-15T23:16:22,090 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/fb4910cbde6343ffa4c7cb1c3a6fbba3 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/fb4910cbde6343ffa4c7cb1c3a6fbba3 2024-11-15T23:16:22,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742128_1304 (size=13085) 2024-11-15T23:16:22,100 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/B of 083d01f7dba258590a0a4db0eb54177d into fb4910cbde6343ffa4c7cb1c3a6fbba3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:22,100 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:22,100 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/B, priority=12, startTime=1731712581998; duration=0sec 2024-11-15T23:16:22,100 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:22,100 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:B 2024-11-15T23:16:22,100 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:16:22,102 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44932 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:16:22,102 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/C is initiating minor compaction (all files) 2024-11-15T23:16:22,102 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/C in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:22,102 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/571b1de536324b3ebaa70e2f20551ae0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/602fa765ff724d4582e351df24ac5c14, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/8937567aa8344302aa4d359e50dff17f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/f5afcf9f736b465cae6f937363da4201] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=43.9 K 2024-11-15T23:16:22,105 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 571b1de536324b3ebaa70e2f20551ae0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731712576986 2024-11-15T23:16:22,106 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/19a0b2c5d3824dd6b1b818b02e47490e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/19a0b2c5d3824dd6b1b818b02e47490e 2024-11-15T23:16:22,106 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 602fa765ff724d4582e351df24ac5c14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731712577691 2024-11-15T23:16:22,107 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 8937567aa8344302aa4d359e50dff17f, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731712578847 2024-11-15T23:16:22,108 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting f5afcf9f736b465cae6f937363da4201, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731712581023 2024-11-15T23:16:22,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T23:16:22,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:22,110 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-15T23:16:22,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:22,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:22,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:22,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:22,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:22,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:22,112 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/A of 083d01f7dba258590a0a4db0eb54177d into 19a0b2c5d3824dd6b1b818b02e47490e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:22,112 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:22,112 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/A, priority=12, startTime=1731712581998; duration=0sec 2024-11-15T23:16:22,112 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:22,112 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:A 2024-11-15T23:16:22,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/109b5f5f45fd40878a40dc3af7de2a23 is 50, key is test_row_0/A:col10/1731712581092/Put/seqid=0 2024-11-15T23:16:22,138 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#C#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:22,139 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/59059992cf55474f8f22769b93ed47a8 is 50, key is test_row_0/C:col10/1731712581029/Put/seqid=0 2024-11-15T23:16:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T23:16:22,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742129_1305 (size=12301) 2024-11-15T23:16:22,186 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/109b5f5f45fd40878a40dc3af7de2a23 2024-11-15T23:16:22,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742130_1306 (size=13051) 2024-11-15T23:16:22,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/87c980b31e1b43eea81fa6d75335bda8 is 50, key is test_row_0/B:col10/1731712581092/Put/seqid=0 2024-11-15T23:16:22,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742131_1307 (size=12301) 2024-11-15T23:16:22,209 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/87c980b31e1b43eea81fa6d75335bda8 2024-11-15T23:16:22,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/8bd98bd487b94e9a92a91a066ddb695b is 50, key is test_row_0/C:col10/1731712581092/Put/seqid=0 2024-11-15T23:16:22,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742132_1308 (size=12301) 2024-11-15T23:16:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:22,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:22,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712642274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:22,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712642274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:22,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:22,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:22,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712642276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:22,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:22,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712642385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:22,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:22,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712642385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:22,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712642385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:22,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:22,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:22,594 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/59059992cf55474f8f22769b93ed47a8 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/59059992cf55474f8f22769b93ed47a8 2024-11-15T23:16:22,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712642596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:22,599 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/C of 083d01f7dba258590a0a4db0eb54177d into 59059992cf55474f8f22769b93ed47a8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:22,599 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:22,599 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/C, priority=12, startTime=1731712581998; duration=0sec 2024-11-15T23:16:22,599 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:22,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,599 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:C 2024-11-15T23:16:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712642596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:22,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:22,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:22,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712642596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:22,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:22,628 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/8bd98bd487b94e9a92a91a066ddb695b 2024-11-15T23:16:22,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/109b5f5f45fd40878a40dc3af7de2a23 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/109b5f5f45fd40878a40dc3af7de2a23 2024-11-15T23:16:22,638 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/109b5f5f45fd40878a40dc3af7de2a23, entries=150, sequenceid=383, filesize=12.0 K 2024-11-15T23:16:22,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/87c980b31e1b43eea81fa6d75335bda8 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/87c980b31e1b43eea81fa6d75335bda8 2024-11-15T23:16:22,644 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/87c980b31e1b43eea81fa6d75335bda8, entries=150, sequenceid=383, filesize=12.0 K 2024-11-15T23:16:22,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/8bd98bd487b94e9a92a91a066ddb695b as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/8bd98bd487b94e9a92a91a066ddb695b 2024-11-15T23:16:22,650 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/8bd98bd487b94e9a92a91a066ddb695b, entries=150, sequenceid=383, filesize=12.0 K 2024-11-15T23:16:22,651 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 083d01f7dba258590a0a4db0eb54177d in 541ms, sequenceid=383, compaction requested=false 2024-11-15T23:16:22,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:22,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:22,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-15T23:16:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-15T23:16:22,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-15T23:16:22,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6220 sec 2024-11-15T23:16:22,656 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.6270 sec 2024-11-15T23:16:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:22,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T23:16:22,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:22,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:22,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:22,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:22,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:22,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:22,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/0f1d64017e6247c288ee9bf22b5f559b is 50, key is test_row_0/A:col10/1731712582907/Put/seqid=0 2024-11-15T23:16:22,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742133_1309 (size=17181) 2024-11-15T23:16:22,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/0f1d64017e6247c288ee9bf22b5f559b 2024-11-15T23:16:22,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/65da93322bf445d3b47f5665e47ddd27 is 50, key is test_row_0/B:col10/1731712582907/Put/seqid=0 2024-11-15T23:16:22,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742134_1310 (size=12301) 2024-11-15T23:16:22,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/65da93322bf445d3b47f5665e47ddd27 2024-11-15T23:16:22,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712642929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:22,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:22,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712642931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712642932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:22,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/c8b5e5a84d1a4d1295d0fc96830a862c is 50, key is test_row_0/C:col10/1731712582907/Put/seqid=0 2024-11-15T23:16:22,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:22,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:22,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742135_1311 (size=12301) 2024-11-15T23:16:23,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712643045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:23,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712643045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712643045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:23,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T23:16:23,175 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:23,176 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:16:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-15T23:16:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T23:16:23,179 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:16:23,179 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:16:23,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:16:23,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712643255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712643255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712643255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T23:16:23,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T23:16:23,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:23,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:23,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:23,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:23,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:23,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/c8b5e5a84d1a4d1295d0fc96830a862c 2024-11-15T23:16:23,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/0f1d64017e6247c288ee9bf22b5f559b as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/0f1d64017e6247c288ee9bf22b5f559b 2024-11-15T23:16:23,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/0f1d64017e6247c288ee9bf22b5f559b, entries=250, sequenceid=399, filesize=16.8 K 2024-11-15T23:16:23,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/65da93322bf445d3b47f5665e47ddd27 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/65da93322bf445d3b47f5665e47ddd27 2024-11-15T23:16:23,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/65da93322bf445d3b47f5665e47ddd27, entries=150, sequenceid=399, filesize=12.0 K 2024-11-15T23:16:23,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/c8b5e5a84d1a4d1295d0fc96830a862c as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/c8b5e5a84d1a4d1295d0fc96830a862c 2024-11-15T23:16:23,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/c8b5e5a84d1a4d1295d0fc96830a862c, entries=150, sequenceid=399, filesize=12.0 K 2024-11-15T23:16:23,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 083d01f7dba258590a0a4db0eb54177d in 449ms, sequenceid=399, compaction requested=true 2024-11-15T23:16:23,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:23,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:16:23,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:23,356 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:23,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:16:23,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:23,356 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:23,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:16:23,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:23,357 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:23,357 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/B is initiating minor compaction (all files) 2024-11-15T23:16:23,357 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/B in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:23,357 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/fb4910cbde6343ffa4c7cb1c3a6fbba3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/87c980b31e1b43eea81fa6d75335bda8, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/65da93322bf445d3b47f5665e47ddd27] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=36.8 K 2024-11-15T23:16:23,358 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42567 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:23,358 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/A is initiating minor compaction (all files) 2024-11-15T23:16:23,358 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/A in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:23,358 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting fb4910cbde6343ffa4c7cb1c3a6fbba3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731712581023 2024-11-15T23:16:23,358 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/19a0b2c5d3824dd6b1b818b02e47490e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/109b5f5f45fd40878a40dc3af7de2a23, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/0f1d64017e6247c288ee9bf22b5f559b] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=41.6 K 2024-11-15T23:16:23,358 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19a0b2c5d3824dd6b1b818b02e47490e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731712581023 2024-11-15T23:16:23,358 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 87c980b31e1b43eea81fa6d75335bda8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731712581092 2024-11-15T23:16:23,358 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 109b5f5f45fd40878a40dc3af7de2a23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731712581092 2024-11-15T23:16:23,359 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 65da93322bf445d3b47f5665e47ddd27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731712582906 2024-11-15T23:16:23,360 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f1d64017e6247c288ee9bf22b5f559b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731712582257 2024-11-15T23:16:23,373 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#B#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:23,374 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/efbe8b8350ab490791c97ac160ab6af9 is 50, key is test_row_0/B:col10/1731712582907/Put/seqid=0 2024-11-15T23:16:23,375 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#A#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:23,376 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/badaef93b6744e37b152cc70ac15e125 is 50, key is test_row_0/A:col10/1731712582907/Put/seqid=0 2024-11-15T23:16:23,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742136_1312 (size=13187) 2024-11-15T23:16:23,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742137_1313 (size=13187) 2024-11-15T23:16:23,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T23:16:23,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:23,484 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T23:16:23,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:23,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:23,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:23,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:23,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:23,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:23,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/f14bb582f3c0426a978145d63c3ff730 is 50, key is test_row_0/A:col10/1731712582929/Put/seqid=0 2024-11-15T23:16:23,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742138_1314 (size=12301) 2024-11-15T23:16:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T23:16:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:23,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:23,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:23,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712643594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:23,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:23,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712643597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:23,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712643597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:23,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712643706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712643706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:23,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712643715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:23,783 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/efbe8b8350ab490791c97ac160ab6af9 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/efbe8b8350ab490791c97ac160ab6af9 2024-11-15T23:16:23,790 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/badaef93b6744e37b152cc70ac15e125 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/badaef93b6744e37b152cc70ac15e125 2024-11-15T23:16:23,797 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/B of 083d01f7dba258590a0a4db0eb54177d into efbe8b8350ab490791c97ac160ab6af9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:23,797 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:23,797 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/B, priority=13, startTime=1731712583356; duration=0sec 2024-11-15T23:16:23,797 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:23,797 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:B 2024-11-15T23:16:23,797 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:23,800 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/A of 083d01f7dba258590a0a4db0eb54177d into badaef93b6744e37b152cc70ac15e125(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:23,800 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:23,800 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/A, priority=13, startTime=1731712583356; duration=0sec 2024-11-15T23:16:23,800 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:23,800 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:A 2024-11-15T23:16:23,801 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:23,801 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/C is initiating minor compaction (all files) 2024-11-15T23:16:23,801 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/C in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:23,801 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/59059992cf55474f8f22769b93ed47a8, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/8bd98bd487b94e9a92a91a066ddb695b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/c8b5e5a84d1a4d1295d0fc96830a862c] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=36.8 K 2024-11-15T23:16:23,801 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 59059992cf55474f8f22769b93ed47a8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731712581023 2024-11-15T23:16:23,801 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bd98bd487b94e9a92a91a066ddb695b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731712581092 2024-11-15T23:16:23,802 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting c8b5e5a84d1a4d1295d0fc96830a862c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731712582906 2024-11-15T23:16:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T23:16:23,809 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#C#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:23,809 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/a20b256acbfa429f887026e015312716 is 50, key is test_row_0/C:col10/1731712582907/Put/seqid=0 2024-11-15T23:16:23,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742139_1315 (size=13153) 2024-11-15T23:16:23,895 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/f14bb582f3c0426a978145d63c3ff730 2024-11-15T23:16:23,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/dc25a7fed94942d2afcbd0e11b6b2804 is 50, key is test_row_0/B:col10/1731712582929/Put/seqid=0 2024-11-15T23:16:23,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712643916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:23,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712643916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:23,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712643926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:23,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742140_1316 (size=12301) 2024-11-15T23:16:23,928 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/dc25a7fed94942d2afcbd0e11b6b2804 2024-11-15T23:16:23,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/ad0c06fb34ed46efa07064157ce5fca3 is 50, key is test_row_0/C:col10/1731712582929/Put/seqid=0 2024-11-15T23:16:23,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742141_1317 (size=12301) 2024-11-15T23:16:23,975 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/ad0c06fb34ed46efa07064157ce5fca3 2024-11-15T23:16:23,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/f14bb582f3c0426a978145d63c3ff730 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/f14bb582f3c0426a978145d63c3ff730 2024-11-15T23:16:23,993 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/f14bb582f3c0426a978145d63c3ff730, entries=150, sequenceid=417, filesize=12.0 K 2024-11-15T23:16:23,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/dc25a7fed94942d2afcbd0e11b6b2804 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/dc25a7fed94942d2afcbd0e11b6b2804 2024-11-15T23:16:24,002 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/dc25a7fed94942d2afcbd0e11b6b2804, entries=150, sequenceid=417, filesize=12.0 K 2024-11-15T23:16:24,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/ad0c06fb34ed46efa07064157ce5fca3 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/ad0c06fb34ed46efa07064157ce5fca3 2024-11-15T23:16:24,028 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/ad0c06fb34ed46efa07064157ce5fca3, entries=150, sequenceid=417, filesize=12.0 K 2024-11-15T23:16:24,029 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 083d01f7dba258590a0a4db0eb54177d in 545ms, sequenceid=417, compaction requested=false 2024-11-15T23:16:24,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:24,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:24,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-15T23:16:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-15T23:16:24,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-15T23:16:24,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 852 msec 2024-11-15T23:16:24,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 857 msec 2024-11-15T23:16:24,229 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/a20b256acbfa429f887026e015312716 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/a20b256acbfa429f887026e015312716 2024-11-15T23:16:24,237 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/C of 083d01f7dba258590a0a4db0eb54177d into a20b256acbfa429f887026e015312716(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:24,237 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:24,237 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/C, priority=13, startTime=1731712583356; duration=0sec 2024-11-15T23:16:24,237 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:24,237 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:C 2024-11-15T23:16:24,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T23:16:24,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:24,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:24,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:24,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:24,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:24,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:24,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:24,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/025ff370efb14da790d80ca71eeb57c5 is 50, key is test_row_0/A:col10/1731712583594/Put/seqid=0 2024-11-15T23:16:24,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712644268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:24,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742142_1318 (size=12301) 2024-11-15T23:16:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:24,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/025ff370efb14da790d80ca71eeb57c5 2024-11-15T23:16:24,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712644274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:24,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:24,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712644275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:24,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/36ff9bd2ac414ecd8bfd9e32ec071b35 is 50, key is test_row_0/B:col10/1731712583594/Put/seqid=0 2024-11-15T23:16:24,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742143_1319 (size=12301) 2024-11-15T23:16:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T23:16:24,316 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:24,316 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:16:24,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-15T23:16:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T23:16:24,318 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:16:24,319 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:16:24,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:16:24,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712644385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:24,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712644385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:24,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712644385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T23:16:24,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T23:16:24,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:24,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:24,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:24,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:24,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:24,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712644595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:24,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712644595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712644595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:24,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T23:16:24,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:24,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:24,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:24,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:24,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T23:16:24,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/36ff9bd2ac414ecd8bfd9e32ec071b35 2024-11-15T23:16:24,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/696d0b2c764a4b859b27c4c49053880e is 50, key is test_row_0/C:col10/1731712583594/Put/seqid=0 2024-11-15T23:16:24,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742144_1320 (size=12301) 2024-11-15T23:16:24,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T23:16:24,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:24,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:24,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:24,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:24,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:24,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712644905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:24,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712644905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712644906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:24,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T23:16:24,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:24,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:24,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:24,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:24,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T23:16:25,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T23:16:25,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:25,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:25,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:25,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:25,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:25,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/696d0b2c764a4b859b27c4c49053880e 2024-11-15T23:16:25,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/025ff370efb14da790d80ca71eeb57c5 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/025ff370efb14da790d80ca71eeb57c5 2024-11-15T23:16:25,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/025ff370efb14da790d80ca71eeb57c5, entries=150, sequenceid=437, filesize=12.0 K 2024-11-15T23:16:25,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/36ff9bd2ac414ecd8bfd9e32ec071b35 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/36ff9bd2ac414ecd8bfd9e32ec071b35 2024-11-15T23:16:25,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/36ff9bd2ac414ecd8bfd9e32ec071b35, entries=150, sequenceid=437, filesize=12.0 K 2024-11-15T23:16:25,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/696d0b2c764a4b859b27c4c49053880e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/696d0b2c764a4b859b27c4c49053880e 2024-11-15T23:16:25,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/696d0b2c764a4b859b27c4c49053880e, entries=150, sequenceid=437, filesize=12.0 K 2024-11-15T23:16:25,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 083d01f7dba258590a0a4db0eb54177d in 880ms, sequenceid=437, compaction requested=true 2024-11-15T23:16:25,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:25,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:16:25,118 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:25,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:25,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:16:25,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:25,118 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:25,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:16:25,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:25,119 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:25,119 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/B is initiating minor compaction (all files) 2024-11-15T23:16:25,119 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/B in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:25,119 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/efbe8b8350ab490791c97ac160ab6af9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/dc25a7fed94942d2afcbd0e11b6b2804, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/36ff9bd2ac414ecd8bfd9e32ec071b35] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=36.9 K 2024-11-15T23:16:25,120 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting efbe8b8350ab490791c97ac160ab6af9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731712582906 2024-11-15T23:16:25,121 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:25,121 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/A is initiating minor compaction (all files) 2024-11-15T23:16:25,121 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting dc25a7fed94942d2afcbd0e11b6b2804, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731712582926 2024-11-15T23:16:25,121 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/A in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:25,121 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/badaef93b6744e37b152cc70ac15e125, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/f14bb582f3c0426a978145d63c3ff730, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/025ff370efb14da790d80ca71eeb57c5] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=36.9 K 2024-11-15T23:16:25,121 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting badaef93b6744e37b152cc70ac15e125, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731712582906 2024-11-15T23:16:25,121 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 36ff9bd2ac414ecd8bfd9e32ec071b35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731712583594 2024-11-15T23:16:25,122 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting f14bb582f3c0426a978145d63c3ff730, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731712582926 2024-11-15T23:16:25,122 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 025ff370efb14da790d80ca71eeb57c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731712583594 2024-11-15T23:16:25,146 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#A#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:25,147 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/85bcbb501ac5420bbeda4acdcd5ad542 is 50, key is test_row_0/A:col10/1731712583594/Put/seqid=0 2024-11-15T23:16:25,150 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#B#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:25,150 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/8c9bde5405ee4ab393d18770458bc8ce is 50, key is test_row_0/B:col10/1731712583594/Put/seqid=0 2024-11-15T23:16:25,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742145_1321 (size=13289) 2024-11-15T23:16:25,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742146_1322 (size=13289) 2024-11-15T23:16:25,196 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/8c9bde5405ee4ab393d18770458bc8ce as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/8c9bde5405ee4ab393d18770458bc8ce 2024-11-15T23:16:25,203 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/B of 083d01f7dba258590a0a4db0eb54177d into 8c9bde5405ee4ab393d18770458bc8ce(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:25,203 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:25,203 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/B, priority=13, startTime=1731712585118; duration=0sec 2024-11-15T23:16:25,203 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:25,203 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:B 2024-11-15T23:16:25,203 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:25,204 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:25,205 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/C is initiating minor compaction (all files) 2024-11-15T23:16:25,205 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/C in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:25,205 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/a20b256acbfa429f887026e015312716, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/ad0c06fb34ed46efa07064157ce5fca3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/696d0b2c764a4b859b27c4c49053880e] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=36.9 K 2024-11-15T23:16:25,205 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting a20b256acbfa429f887026e015312716, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731712582906 2024-11-15T23:16:25,206 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting ad0c06fb34ed46efa07064157ce5fca3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731712582926 2024-11-15T23:16:25,206 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 696d0b2c764a4b859b27c4c49053880e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731712583594 2024-11-15T23:16:25,212 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#C#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:25,213 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/e166ae03f25943f5a1224a5e66958447 is 50, key is test_row_0/C:col10/1731712583594/Put/seqid=0 2024-11-15T23:16:25,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742147_1323 (size=13255) 2024-11-15T23:16:25,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T23:16:25,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:25,240 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T23:16:25,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:25,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:25,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:25,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:25,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:25,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:25,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/b43dccc2455f492e81ddafb8b4b3f962 is 50, key is test_row_0/A:col10/1731712584273/Put/seqid=0 2024-11-15T23:16:25,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742148_1324 (size=12301) 2024-11-15T23:16:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:25,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. as already flushing 2024-11-15T23:16:25,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712645443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:25,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:25,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712645445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712645445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T23:16:25,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712645556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:25,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:25,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712645556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:25,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:25,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712645556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:25,595 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/85bcbb501ac5420bbeda4acdcd5ad542 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/85bcbb501ac5420bbeda4acdcd5ad542 2024-11-15T23:16:25,601 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/A of 083d01f7dba258590a0a4db0eb54177d into 85bcbb501ac5420bbeda4acdcd5ad542(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:25,601 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:25,601 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/A, priority=13, startTime=1731712585117; duration=0sec 2024-11-15T23:16:25,601 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:25,601 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:A 2024-11-15T23:16:25,625 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/e166ae03f25943f5a1224a5e66958447 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/e166ae03f25943f5a1224a5e66958447 2024-11-15T23:16:25,629 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/C of 083d01f7dba258590a0a4db0eb54177d into e166ae03f25943f5a1224a5e66958447(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:25,629 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:25,629 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/C, priority=13, startTime=1731712585118; duration=0sec 2024-11-15T23:16:25,629 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:25,629 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:C 2024-11-15T23:16:25,649 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/b43dccc2455f492e81ddafb8b4b3f962 2024-11-15T23:16:25,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/497b22d5adcd43b2b50e3df4834d495e is 50, key is test_row_0/B:col10/1731712584273/Put/seqid=0 2024-11-15T23:16:25,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742149_1325 (size=12301) 2024-11-15T23:16:25,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712645765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:25,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712645765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:25,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712645765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:25,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:25,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51904 deadline: 1731712645905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:25,908 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in 083d01f7dba258590a0a4db0eb54177d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:16:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:25,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:25,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51892 deadline: 1731712645936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:25,938 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_0' in 083d01f7dba258590a0a4db0eb54177d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:16:25,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:25,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:26,073 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/497b22d5adcd43b2b50e3df4834d495e 2024-11-15T23:16:26,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712646076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712646076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:26,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/b71e014b1dad4c48ad5bd648e8f782bf is 50, key is test_row_0/C:col10/1731712584273/Put/seqid=0 2024-11-15T23:16:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712646076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:26,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742150_1326 (size=12301) 2024-11-15T23:16:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T23:16:26,483 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/b71e014b1dad4c48ad5bd648e8f782bf 2024-11-15T23:16:26,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/b43dccc2455f492e81ddafb8b4b3f962 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/b43dccc2455f492e81ddafb8b4b3f962 2024-11-15T23:16:26,491 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/b43dccc2455f492e81ddafb8b4b3f962, entries=150, sequenceid=454, filesize=12.0 K 2024-11-15T23:16:26,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/497b22d5adcd43b2b50e3df4834d495e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/497b22d5adcd43b2b50e3df4834d495e 2024-11-15T23:16:26,495 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/497b22d5adcd43b2b50e3df4834d495e, entries=150, sequenceid=454, filesize=12.0 K 2024-11-15T23:16:26,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/b71e014b1dad4c48ad5bd648e8f782bf as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/b71e014b1dad4c48ad5bd648e8f782bf 2024-11-15T23:16:26,500 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/b71e014b1dad4c48ad5bd648e8f782bf, entries=150, sequenceid=454, filesize=12.0 K 2024-11-15T23:16:26,501 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 083d01f7dba258590a0a4db0eb54177d in 1261ms, sequenceid=454, compaction requested=false 2024-11-15T23:16:26,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:26,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:26,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-15T23:16:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-15T23:16:26,504 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-15T23:16:26,504 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1840 sec 2024-11-15T23:16:26,507 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.1890 sec 2024-11-15T23:16:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:26,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T23:16:26,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:26,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:26,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:26,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:26,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:26,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:26,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/fe5d6fc7eedf4c8582b80388a2d1ab9d is 50, key is test_row_0/A:col10/1731712586586/Put/seqid=0 2024-11-15T23:16:26,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742151_1327 (size=14741) 2024-11-15T23:16:26,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712646600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:26,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:26,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712646602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:26,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712646604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:26,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712646715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:26,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712646715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712646716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:26,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712646925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:26,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712646925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712646925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:26,933 INFO [Thread-1112 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1112. 2024-11-15T23:16:26,933 DEBUG [Thread-1112 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:16:26,933 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:26,934 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:26,934 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:26,934 INFO [Thread-1110 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1110. 2024-11-15T23:16:26,934 DEBUG [Thread-1110 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:16:26,935 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:26,935 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:26,935 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:26,935 INFO [Thread-1114 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1114. 2024-11-15T23:16:26,935 DEBUG [Thread-1114 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:16:26,935 DEBUG [Thread-1114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:26,935 DEBUG [Thread-1114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:26,935 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:26,937 INFO [Thread-1116 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1116. 2024-11-15T23:16:26,937 DEBUG [Thread-1116 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:16:26,937 DEBUG [Thread-1116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:26,937 DEBUG [Thread-1116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:26,937 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:26,938 INFO [Thread-1108 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1108. 2024-11-15T23:16:26,938 DEBUG [Thread-1108 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:16:26,938 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:26,938 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:26,938 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:26,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/fe5d6fc7eedf4c8582b80388a2d1ab9d 2024-11-15T23:16:27,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/2c6304a919304f16a92d59bf067b36ed is 50, key is test_row_0/B:col10/1731712586586/Put/seqid=0 2024-11-15T23:16:27,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742152_1328 (size=12301) 2024-11-15T23:16:27,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:27,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712647236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712647236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:27,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712647237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:27,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:27,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:27,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/2c6304a919304f16a92d59bf067b36ed 2024-11-15T23:16:27,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/31e97a9e39c74705be91f34a4cfaa37a is 50, key is test_row_0/C:col10/1731712586586/Put/seqid=0 2024-11-15T23:16:27,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742153_1329 (size=12301) 2024-11-15T23:16:27,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:27,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:27,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51832 deadline: 1731712647745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51872 deadline: 1731712647745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731712647745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:16:27,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/31e97a9e39c74705be91f34a4cfaa37a 2024-11-15T23:16:27,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/fe5d6fc7eedf4c8582b80388a2d1ab9d as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/fe5d6fc7eedf4c8582b80388a2d1ab9d 2024-11-15T23:16:27,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/fe5d6fc7eedf4c8582b80388a2d1ab9d, entries=200, sequenceid=475, filesize=14.4 K 2024-11-15T23:16:27,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/2c6304a919304f16a92d59bf067b36ed as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2c6304a919304f16a92d59bf067b36ed 2024-11-15T23:16:27,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2c6304a919304f16a92d59bf067b36ed, entries=150, sequenceid=475, filesize=12.0 K 2024-11-15T23:16:27,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/31e97a9e39c74705be91f34a4cfaa37a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/31e97a9e39c74705be91f34a4cfaa37a 2024-11-15T23:16:27,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/31e97a9e39c74705be91f34a4cfaa37a, entries=150, sequenceid=475, filesize=12.0 K 2024-11-15T23:16:27,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 083d01f7dba258590a0a4db0eb54177d in 1254ms, sequenceid=475, compaction requested=true 2024-11-15T23:16:27,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:27,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:16:27,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:27,841 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:27,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:16:27,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:27,841 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:27,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 083d01f7dba258590a0a4db0eb54177d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:16:27,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:27,842 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:27,842 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:27,842 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/A is initiating minor compaction (all files) 2024-11-15T23:16:27,842 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/B is initiating minor compaction (all files) 2024-11-15T23:16:27,842 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/B in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:27,842 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/A in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:27,842 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/8c9bde5405ee4ab393d18770458bc8ce, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/497b22d5adcd43b2b50e3df4834d495e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2c6304a919304f16a92d59bf067b36ed] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=37.0 K 2024-11-15T23:16:27,842 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/85bcbb501ac5420bbeda4acdcd5ad542, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/b43dccc2455f492e81ddafb8b4b3f962, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/fe5d6fc7eedf4c8582b80388a2d1ab9d] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=39.4 K 2024-11-15T23:16:27,843 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85bcbb501ac5420bbeda4acdcd5ad542, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731712583594 2024-11-15T23:16:27,843 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c9bde5405ee4ab393d18770458bc8ce, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731712583594 2024-11-15T23:16:27,843 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting b43dccc2455f492e81ddafb8b4b3f962, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731712584267 2024-11-15T23:16:27,843 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 497b22d5adcd43b2b50e3df4834d495e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731712584267 2024-11-15T23:16:27,843 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c6304a919304f16a92d59bf067b36ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731712585443 2024-11-15T23:16:27,843 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe5d6fc7eedf4c8582b80388a2d1ab9d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731712585423 2024-11-15T23:16:27,850 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#A#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:27,850 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:27,850 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/ae28aeaa4bfd4d2d84cab1bd41cdef73 is 50, key is test_row_0/A:col10/1731712586586/Put/seqid=0 2024-11-15T23:16:27,851 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/841542dce88343248f578efbb4393f9a is 50, key is test_row_0/B:col10/1731712586586/Put/seqid=0 2024-11-15T23:16:27,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742155_1331 (size=13391) 2024-11-15T23:16:27,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742154_1330 (size=13391) 2024-11-15T23:16:27,870 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/841542dce88343248f578efbb4393f9a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/841542dce88343248f578efbb4393f9a 2024-11-15T23:16:27,870 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/ae28aeaa4bfd4d2d84cab1bd41cdef73 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/ae28aeaa4bfd4d2d84cab1bd41cdef73 2024-11-15T23:16:27,874 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/B of 083d01f7dba258590a0a4db0eb54177d into 841542dce88343248f578efbb4393f9a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:27,874 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/A of 083d01f7dba258590a0a4db0eb54177d into ae28aeaa4bfd4d2d84cab1bd41cdef73(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:27,874 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:27,874 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:27,874 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/B, priority=13, startTime=1731712587841; duration=0sec 2024-11-15T23:16:27,874 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/A, priority=13, startTime=1731712587841; duration=0sec 2024-11-15T23:16:27,874 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:27,874 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:27,874 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:A 2024-11-15T23:16:27,874 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:B 2024-11-15T23:16:27,874 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:27,875 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:27,875 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 083d01f7dba258590a0a4db0eb54177d/C is initiating minor compaction (all files) 2024-11-15T23:16:27,875 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 083d01f7dba258590a0a4db0eb54177d/C in TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:27,875 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/e166ae03f25943f5a1224a5e66958447, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/b71e014b1dad4c48ad5bd648e8f782bf, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/31e97a9e39c74705be91f34a4cfaa37a] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp, totalSize=37.0 K 2024-11-15T23:16:27,875 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting e166ae03f25943f5a1224a5e66958447, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731712583594 2024-11-15T23:16:27,876 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting b71e014b1dad4c48ad5bd648e8f782bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731712584267 2024-11-15T23:16:27,876 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31e97a9e39c74705be91f34a4cfaa37a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731712585443 2024-11-15T23:16:27,882 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 083d01f7dba258590a0a4db0eb54177d#C#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:27,882 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/3f889aaef89b486ab3b17930a14d8867 is 50, key is test_row_0/C:col10/1731712586586/Put/seqid=0 2024-11-15T23:16:27,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742156_1332 (size=13357) 2024-11-15T23:16:28,292 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/3f889aaef89b486ab3b17930a14d8867 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/3f889aaef89b486ab3b17930a14d8867 2024-11-15T23:16:28,298 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 083d01f7dba258590a0a4db0eb54177d/C of 083d01f7dba258590a0a4db0eb54177d into 3f889aaef89b486ab3b17930a14d8867(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:28,298 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:28,298 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., storeName=083d01f7dba258590a0a4db0eb54177d/C, priority=13, startTime=1731712587841; duration=0sec 2024-11-15T23:16:28,298 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:28,298 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 083d01f7dba258590a0a4db0eb54177d:C 2024-11-15T23:16:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T23:16:28,487 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:28,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T23:16:28,762 INFO [Thread-1099 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1099. 2024-11-15T23:16:28,762 INFO [Thread-1103 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1103. 2024-11-15T23:16:28,762 INFO [Thread-1097 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1097. 2024-11-15T23:16:28,762 DEBUG [Thread-1099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:16:28,762 DEBUG [Thread-1097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:16:28,762 DEBUG [Thread-1103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:16:28,762 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:28,762 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:28,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:28,762 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:28,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:28,762 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:28,762 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:28,762 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:28,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:28,762 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:28,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:28,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:28,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:28,762 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:28,762 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:28,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/d47a8cf01da54e269eea4f2f4ca05b0a is 50, key is test_row_0/A:col10/1731712586601/Put/seqid=0 2024-11-15T23:16:28,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742157_1333 (size=12301) 2024-11-15T23:16:29,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/d47a8cf01da54e269eea4f2f4ca05b0a 2024-11-15T23:16:29,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/7a8ce3d2ff72457e97908d698132e481 is 50, key is test_row_0/B:col10/1731712586601/Put/seqid=0 2024-11-15T23:16:29,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742158_1334 (size=12301) 2024-11-15T23:16:29,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/7a8ce3d2ff72457e97908d698132e481 2024-11-15T23:16:29,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/dd3e5d20a9674aef9888f8b4dddf1a62 is 50, key is test_row_0/C:col10/1731712586601/Put/seqid=0 2024-11-15T23:16:29,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742159_1335 (size=12301) 2024-11-15T23:16:30,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/dd3e5d20a9674aef9888f8b4dddf1a62 2024-11-15T23:16:30,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/d47a8cf01da54e269eea4f2f4ca05b0a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/d47a8cf01da54e269eea4f2f4ca05b0a 2024-11-15T23:16:30,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/d47a8cf01da54e269eea4f2f4ca05b0a, entries=150, sequenceid=494, filesize=12.0 K 2024-11-15T23:16:30,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/7a8ce3d2ff72457e97908d698132e481 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/7a8ce3d2ff72457e97908d698132e481 2024-11-15T23:16:30,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/7a8ce3d2ff72457e97908d698132e481, entries=150, sequenceid=494, filesize=12.0 K 2024-11-15T23:16:30,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/dd3e5d20a9674aef9888f8b4dddf1a62 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/dd3e5d20a9674aef9888f8b4dddf1a62 2024-11-15T23:16:30,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/dd3e5d20a9674aef9888f8b4dddf1a62, entries=150, sequenceid=494, filesize=12.0 K 2024-11-15T23:16:30,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 083d01f7dba258590a0a4db0eb54177d in 1265ms, sequenceid=494, compaction requested=false 2024-11-15T23:16:30,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 083d01f7dba258590a0a4db0eb54177d: 2024-11-15T23:16:31,968 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T23:16:35,980 INFO [Thread-1101 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1101. 2024-11-15T23:16:35,981 DEBUG [Thread-1101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:16:35,981 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:35,981 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:35,981 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:36,016 INFO [Thread-1105 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1105. 2024-11-15T23:16:36,016 DEBUG [Thread-1105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:16:36,016 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:36,016 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:36,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:36,017 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T23:16:36,017 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-15T23:16:36,017 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-15T23:16:36,017 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-15T23:16:36,017 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-15T23:16:36,017 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-15T23:16:36,017 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T23:16:36,017 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3230 2024-11-15T23:16:36,017 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3207 2024-11-15T23:16:36,017 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3065 2024-11-15T23:16:36,017 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3232 2024-11-15T23:16:36,017 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3195 2024-11-15T23:16:36,017 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T23:16:36,017 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T23:16:36,017 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T23:16:36,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:36,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:36,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:16:36,018 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T23:16:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T23:16:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T23:16:36,021 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712596020"}]},"ts":"1731712596020"} 2024-11-15T23:16:36,022 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T23:16:36,022 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T23:16:36,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T23:16:36,025 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=083d01f7dba258590a0a4db0eb54177d, UNASSIGN}] 2024-11-15T23:16:36,025 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=083d01f7dba258590a0a4db0eb54177d, UNASSIGN 2024-11-15T23:16:36,026 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=083d01f7dba258590a0a4db0eb54177d, regionState=CLOSING, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:36,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=083d01f7dba258590a0a4db0eb54177d, UNASSIGN because future has completed 2024-11-15T23:16:36,028 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T23:16:36,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763}] 2024-11-15T23:16:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T23:16:36,180 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:36,180 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T23:16:36,181 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing 083d01f7dba258590a0a4db0eb54177d, disabling compactions & flushes 2024-11-15T23:16:36,181 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:36,181 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:36,181 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. after waiting 0 ms 2024-11-15T23:16:36,181 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:36,181 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing 083d01f7dba258590a0a4db0eb54177d 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-15T23:16:36,182 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=A 2024-11-15T23:16:36,182 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:36,182 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=B 2024-11-15T23:16:36,182 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:36,182 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 083d01f7dba258590a0a4db0eb54177d, store=C 2024-11-15T23:16:36,182 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:36,189 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/6f54b2bc319149bc893ba9271a482843 is 50, key is test_row_0/A:col10/1731712596015/Put/seqid=0 2024-11-15T23:16:36,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742160_1336 (size=9857) 2024-11-15T23:16:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T23:16:36,597 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/6f54b2bc319149bc893ba9271a482843 2024-11-15T23:16:36,603 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/7f92c1954b6b46a6bddc836dc8a8c745 is 50, key is test_row_0/B:col10/1731712596015/Put/seqid=0 2024-11-15T23:16:36,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742161_1337 (size=9857) 2024-11-15T23:16:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T23:16:37,007 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/7f92c1954b6b46a6bddc836dc8a8c745 2024-11-15T23:16:37,014 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/57998200f83e4e7bb4e6c4895cbd2fc5 is 50, key is test_row_0/C:col10/1731712596015/Put/seqid=0 2024-11-15T23:16:37,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742162_1338 (size=9857) 2024-11-15T23:16:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T23:16:37,418 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/57998200f83e4e7bb4e6c4895cbd2fc5 2024-11-15T23:16:37,427 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/A/6f54b2bc319149bc893ba9271a482843 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/6f54b2bc319149bc893ba9271a482843 2024-11-15T23:16:37,433 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/6f54b2bc319149bc893ba9271a482843, entries=100, sequenceid=499, filesize=9.6 K 2024-11-15T23:16:37,434 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/B/7f92c1954b6b46a6bddc836dc8a8c745 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/7f92c1954b6b46a6bddc836dc8a8c745 2024-11-15T23:16:37,438 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/7f92c1954b6b46a6bddc836dc8a8c745, entries=100, sequenceid=499, filesize=9.6 K 2024-11-15T23:16:37,438 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/.tmp/C/57998200f83e4e7bb4e6c4895cbd2fc5 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/57998200f83e4e7bb4e6c4895cbd2fc5 2024-11-15T23:16:37,441 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/57998200f83e4e7bb4e6c4895cbd2fc5, entries=100, sequenceid=499, filesize=9.6 K 2024-11-15T23:16:37,442 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 083d01f7dba258590a0a4db0eb54177d in 1261ms, sequenceid=499, compaction requested=true 2024-11-15T23:16:37,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/773a5d1b9b6b4e2c969d5223d22c617e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/a2552639ad954511ba6d85dbf2e11b32, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/8a26f4f97b6c4068b4b3fd3a33f14e21, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/c53082910d1e4b76a571cfa5099e9721, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/80633ca5068d45548cdcb4be2f1d1517, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/c667bd939d064d559368abb8c6ed6708, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/a84f9cd8eacc448dbb8ed3537419b652, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/52030626c2414c319b99a70f8b5dfb5e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/dfbf09fa25c140abb27546ec137da749, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/8762d1a9527347ae9b8e9cd44da25cbf, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/a893be96577b48fbbb47854d596a610e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/79cd09324bf4405ba2b5f09d2e7239e4, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/ae125be6dc8a4a13959b49d61c46eec1, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/d0b892a5594f40aabf1cd33286eb9505, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/12e7b00348f24d0dba07281cd9cfccad, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/91757c8c3b8e455f929e66fe448e4f16, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/c30430ab01904d1894b88cea008730e7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/f1eab116062c46c49ad5b03724662114, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/72947a430de746cdbd4c7a1241947b44, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/6d5e707df86e48eaa6be4e41351cb92f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/939cbbc0cbd9446c8f115deb0204d116, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/61e3ba7a90944078bf3e6302e329f944, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/67c3d9eaced94158a8987a18fd36a9b8, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/19a0b2c5d3824dd6b1b818b02e47490e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/109b5f5f45fd40878a40dc3af7de2a23, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/0f1d64017e6247c288ee9bf22b5f559b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/badaef93b6744e37b152cc70ac15e125, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/f14bb582f3c0426a978145d63c3ff730, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/85bcbb501ac5420bbeda4acdcd5ad542, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/025ff370efb14da790d80ca71eeb57c5, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/b43dccc2455f492e81ddafb8b4b3f962, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/fe5d6fc7eedf4c8582b80388a2d1ab9d] to archive 2024-11-15T23:16:37,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:16:37,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/773a5d1b9b6b4e2c969d5223d22c617e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/773a5d1b9b6b4e2c969d5223d22c617e 2024-11-15T23:16:37,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/a2552639ad954511ba6d85dbf2e11b32 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/a2552639ad954511ba6d85dbf2e11b32 2024-11-15T23:16:37,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/8a26f4f97b6c4068b4b3fd3a33f14e21 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/8a26f4f97b6c4068b4b3fd3a33f14e21 2024-11-15T23:16:37,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/c53082910d1e4b76a571cfa5099e9721 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/c53082910d1e4b76a571cfa5099e9721 2024-11-15T23:16:37,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/80633ca5068d45548cdcb4be2f1d1517 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/80633ca5068d45548cdcb4be2f1d1517 2024-11-15T23:16:37,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/c667bd939d064d559368abb8c6ed6708 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/c667bd939d064d559368abb8c6ed6708 2024-11-15T23:16:37,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/a84f9cd8eacc448dbb8ed3537419b652 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/a84f9cd8eacc448dbb8ed3537419b652 2024-11-15T23:16:37,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/52030626c2414c319b99a70f8b5dfb5e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/52030626c2414c319b99a70f8b5dfb5e 2024-11-15T23:16:37,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/dfbf09fa25c140abb27546ec137da749 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/dfbf09fa25c140abb27546ec137da749 2024-11-15T23:16:37,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/8762d1a9527347ae9b8e9cd44da25cbf to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/8762d1a9527347ae9b8e9cd44da25cbf 2024-11-15T23:16:37,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/a893be96577b48fbbb47854d596a610e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/a893be96577b48fbbb47854d596a610e 2024-11-15T23:16:37,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/79cd09324bf4405ba2b5f09d2e7239e4 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/79cd09324bf4405ba2b5f09d2e7239e4 2024-11-15T23:16:37,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/ae125be6dc8a4a13959b49d61c46eec1 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/ae125be6dc8a4a13959b49d61c46eec1 2024-11-15T23:16:37,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/d0b892a5594f40aabf1cd33286eb9505 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/d0b892a5594f40aabf1cd33286eb9505 2024-11-15T23:16:37,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/12e7b00348f24d0dba07281cd9cfccad to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/12e7b00348f24d0dba07281cd9cfccad 2024-11-15T23:16:37,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/91757c8c3b8e455f929e66fe448e4f16 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/91757c8c3b8e455f929e66fe448e4f16 2024-11-15T23:16:37,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/c30430ab01904d1894b88cea008730e7 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/c30430ab01904d1894b88cea008730e7 2024-11-15T23:16:37,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/f1eab116062c46c49ad5b03724662114 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/f1eab116062c46c49ad5b03724662114 2024-11-15T23:16:37,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/72947a430de746cdbd4c7a1241947b44 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/72947a430de746cdbd4c7a1241947b44 2024-11-15T23:16:37,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/6d5e707df86e48eaa6be4e41351cb92f to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/6d5e707df86e48eaa6be4e41351cb92f 2024-11-15T23:16:37,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/939cbbc0cbd9446c8f115deb0204d116 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/939cbbc0cbd9446c8f115deb0204d116 2024-11-15T23:16:37,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/61e3ba7a90944078bf3e6302e329f944 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/61e3ba7a90944078bf3e6302e329f944 2024-11-15T23:16:37,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/67c3d9eaced94158a8987a18fd36a9b8 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/67c3d9eaced94158a8987a18fd36a9b8 2024-11-15T23:16:37,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/19a0b2c5d3824dd6b1b818b02e47490e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/19a0b2c5d3824dd6b1b818b02e47490e 2024-11-15T23:16:37,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/109b5f5f45fd40878a40dc3af7de2a23 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/109b5f5f45fd40878a40dc3af7de2a23 2024-11-15T23:16:37,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/0f1d64017e6247c288ee9bf22b5f559b to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/0f1d64017e6247c288ee9bf22b5f559b 2024-11-15T23:16:37,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/badaef93b6744e37b152cc70ac15e125 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/badaef93b6744e37b152cc70ac15e125 2024-11-15T23:16:37,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/f14bb582f3c0426a978145d63c3ff730 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/f14bb582f3c0426a978145d63c3ff730 2024-11-15T23:16:37,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/85bcbb501ac5420bbeda4acdcd5ad542 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/85bcbb501ac5420bbeda4acdcd5ad542 2024-11-15T23:16:37,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/025ff370efb14da790d80ca71eeb57c5 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/025ff370efb14da790d80ca71eeb57c5 2024-11-15T23:16:37,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/b43dccc2455f492e81ddafb8b4b3f962 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/b43dccc2455f492e81ddafb8b4b3f962 2024-11-15T23:16:37,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/fe5d6fc7eedf4c8582b80388a2d1ab9d to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/fe5d6fc7eedf4c8582b80388a2d1ab9d 2024-11-15T23:16:37,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/bcf65eb543654c32b723d33d7c40e9ba, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/302d8cdde9dc43b8874bfa7699f983fc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/68c38dc0c8ac44babff389795279fc51, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/5a8367cc6366443fbf0203806ac7b052, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/9eb9ace7c9054b32a2e9ad552cf0e5c3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/c283991331554e60a1f536235cf2d7b4, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2813dfac7488406aaa2dfedfa443a923, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2a7e4d42fba640dcaa7e8a6654be36a2, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/da6b53618bcc44fbb302b48a1c276957, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/4005db4634834d738849ec2ed5c5fefa, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/f91f1e203d4e432db0983dca5b960881, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/6f3e277c5e0848c2b47850f595fdfb16, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/fea71a7b560349469e1c447184c3a003, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/f3187a506d7a436eaf652883d4bf7654, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/95fddf6a40824a688a2b7e65b80cef08, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2b75b3cbbfba4af7b2cb6341e7f04ab8, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/9549201f819c47e1a8c52563561124ab, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/4fbc1f23ac5d40e3afd08fb5aee27878, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/9bb3c62e00f84ffc8a8301be952c1b88, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/aa9d39db2ea349759797d0315868488d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/79fda4c3b99f4fe58dd2583a60d3ee32, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/02c81d48fc5441cb8f655ef86fd4b19c, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/fb4910cbde6343ffa4c7cb1c3a6fbba3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/25d6ef218fdd47139319c467e575cf32, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/87c980b31e1b43eea81fa6d75335bda8, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/efbe8b8350ab490791c97ac160ab6af9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/65da93322bf445d3b47f5665e47ddd27, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/dc25a7fed94942d2afcbd0e11b6b2804, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/8c9bde5405ee4ab393d18770458bc8ce, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/36ff9bd2ac414ecd8bfd9e32ec071b35, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/497b22d5adcd43b2b50e3df4834d495e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2c6304a919304f16a92d59bf067b36ed] to archive 2024-11-15T23:16:37,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:16:37,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/bcf65eb543654c32b723d33d7c40e9ba to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/bcf65eb543654c32b723d33d7c40e9ba 2024-11-15T23:16:37,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/302d8cdde9dc43b8874bfa7699f983fc to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/302d8cdde9dc43b8874bfa7699f983fc 2024-11-15T23:16:37,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/68c38dc0c8ac44babff389795279fc51 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/68c38dc0c8ac44babff389795279fc51 2024-11-15T23:16:37,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/5a8367cc6366443fbf0203806ac7b052 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/5a8367cc6366443fbf0203806ac7b052 2024-11-15T23:16:37,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/9eb9ace7c9054b32a2e9ad552cf0e5c3 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/9eb9ace7c9054b32a2e9ad552cf0e5c3 2024-11-15T23:16:37,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/c283991331554e60a1f536235cf2d7b4 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/c283991331554e60a1f536235cf2d7b4 2024-11-15T23:16:37,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2813dfac7488406aaa2dfedfa443a923 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2813dfac7488406aaa2dfedfa443a923 2024-11-15T23:16:37,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2a7e4d42fba640dcaa7e8a6654be36a2 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2a7e4d42fba640dcaa7e8a6654be36a2 2024-11-15T23:16:37,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/da6b53618bcc44fbb302b48a1c276957 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/da6b53618bcc44fbb302b48a1c276957 2024-11-15T23:16:37,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/4005db4634834d738849ec2ed5c5fefa to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/4005db4634834d738849ec2ed5c5fefa 2024-11-15T23:16:37,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/f91f1e203d4e432db0983dca5b960881 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/f91f1e203d4e432db0983dca5b960881 2024-11-15T23:16:37,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/6f3e277c5e0848c2b47850f595fdfb16 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/6f3e277c5e0848c2b47850f595fdfb16 2024-11-15T23:16:37,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/fea71a7b560349469e1c447184c3a003 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/fea71a7b560349469e1c447184c3a003 2024-11-15T23:16:37,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/f3187a506d7a436eaf652883d4bf7654 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/f3187a506d7a436eaf652883d4bf7654 2024-11-15T23:16:37,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/95fddf6a40824a688a2b7e65b80cef08 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/95fddf6a40824a688a2b7e65b80cef08 2024-11-15T23:16:37,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2b75b3cbbfba4af7b2cb6341e7f04ab8 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2b75b3cbbfba4af7b2cb6341e7f04ab8 2024-11-15T23:16:37,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/9549201f819c47e1a8c52563561124ab to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/9549201f819c47e1a8c52563561124ab 2024-11-15T23:16:37,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/4fbc1f23ac5d40e3afd08fb5aee27878 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/4fbc1f23ac5d40e3afd08fb5aee27878 2024-11-15T23:16:37,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/9bb3c62e00f84ffc8a8301be952c1b88 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/9bb3c62e00f84ffc8a8301be952c1b88 2024-11-15T23:16:37,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/aa9d39db2ea349759797d0315868488d to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/aa9d39db2ea349759797d0315868488d 2024-11-15T23:16:37,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/79fda4c3b99f4fe58dd2583a60d3ee32 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/79fda4c3b99f4fe58dd2583a60d3ee32 2024-11-15T23:16:37,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/02c81d48fc5441cb8f655ef86fd4b19c to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/02c81d48fc5441cb8f655ef86fd4b19c 2024-11-15T23:16:37,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/fb4910cbde6343ffa4c7cb1c3a6fbba3 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/fb4910cbde6343ffa4c7cb1c3a6fbba3 2024-11-15T23:16:37,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/25d6ef218fdd47139319c467e575cf32 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/25d6ef218fdd47139319c467e575cf32 2024-11-15T23:16:37,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/87c980b31e1b43eea81fa6d75335bda8 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/87c980b31e1b43eea81fa6d75335bda8 2024-11-15T23:16:37,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/efbe8b8350ab490791c97ac160ab6af9 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/efbe8b8350ab490791c97ac160ab6af9 2024-11-15T23:16:37,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/65da93322bf445d3b47f5665e47ddd27 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/65da93322bf445d3b47f5665e47ddd27 2024-11-15T23:16:37,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/dc25a7fed94942d2afcbd0e11b6b2804 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/dc25a7fed94942d2afcbd0e11b6b2804 2024-11-15T23:16:37,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/8c9bde5405ee4ab393d18770458bc8ce to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/8c9bde5405ee4ab393d18770458bc8ce 2024-11-15T23:16:37,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/36ff9bd2ac414ecd8bfd9e32ec071b35 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/36ff9bd2ac414ecd8bfd9e32ec071b35 2024-11-15T23:16:37,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/497b22d5adcd43b2b50e3df4834d495e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/497b22d5adcd43b2b50e3df4834d495e 2024-11-15T23:16:37,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2c6304a919304f16a92d59bf067b36ed to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/2c6304a919304f16a92d59bf067b36ed 2024-11-15T23:16:37,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/866ef9c2628e4dd991052ad8b18a07c1, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/b81f3c0d99c944b5ac11198ba95156bc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/1eebfe359b884d6493535beafe9cede7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/343bd8548b244574838184fe8c82f839, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/cad29a4b50ea44c7a6e377aeb67beb66, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/1c7705176b9a43cbae955c75696bcaad, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/cbf4d52997f1468aaa5af970efdb9788, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/aabc507ac2744c93bd6e1560a606e775, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/715fb49d25cc4fe794407814fed375b1, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/62080690adad45059c8b8aa5aeee373f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/73cb82066bf6407abf4047ae7f9ba8cd, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/0e5f3681d4fc445fb7d922961bc2133e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/89d693e65d544a13a707cdf5e4563d86, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/c324a6a7db274453ad3f4eae149e0de6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/67a9ae22632e4534b5994ee7ff6c6b64, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/9673c8c260644db29f5973a03b1219a4, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/c9c7a10fff37447c84d1d0c1380a3a02, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/571b1de536324b3ebaa70e2f20551ae0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/2645b6aa1a5949c89129b1ad74f0b50f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/602fa765ff724d4582e351df24ac5c14, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/8937567aa8344302aa4d359e50dff17f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/59059992cf55474f8f22769b93ed47a8, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/f5afcf9f736b465cae6f937363da4201, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/8bd98bd487b94e9a92a91a066ddb695b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/a20b256acbfa429f887026e015312716, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/c8b5e5a84d1a4d1295d0fc96830a862c, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/ad0c06fb34ed46efa07064157ce5fca3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/e166ae03f25943f5a1224a5e66958447, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/696d0b2c764a4b859b27c4c49053880e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/b71e014b1dad4c48ad5bd648e8f782bf, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/31e97a9e39c74705be91f34a4cfaa37a] to archive 2024-11-15T23:16:37,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:16:37,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/866ef9c2628e4dd991052ad8b18a07c1 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/866ef9c2628e4dd991052ad8b18a07c1 2024-11-15T23:16:37,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/b81f3c0d99c944b5ac11198ba95156bc to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/b81f3c0d99c944b5ac11198ba95156bc 2024-11-15T23:16:37,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/1eebfe359b884d6493535beafe9cede7 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/1eebfe359b884d6493535beafe9cede7 2024-11-15T23:16:37,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/343bd8548b244574838184fe8c82f839 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/343bd8548b244574838184fe8c82f839 2024-11-15T23:16:37,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/cad29a4b50ea44c7a6e377aeb67beb66 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/cad29a4b50ea44c7a6e377aeb67beb66 2024-11-15T23:16:37,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/1c7705176b9a43cbae955c75696bcaad to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/1c7705176b9a43cbae955c75696bcaad 2024-11-15T23:16:37,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/cbf4d52997f1468aaa5af970efdb9788 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/cbf4d52997f1468aaa5af970efdb9788 2024-11-15T23:16:37,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/aabc507ac2744c93bd6e1560a606e775 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/aabc507ac2744c93bd6e1560a606e775 2024-11-15T23:16:37,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/715fb49d25cc4fe794407814fed375b1 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/715fb49d25cc4fe794407814fed375b1 2024-11-15T23:16:37,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/62080690adad45059c8b8aa5aeee373f to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/62080690adad45059c8b8aa5aeee373f 2024-11-15T23:16:37,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/73cb82066bf6407abf4047ae7f9ba8cd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/73cb82066bf6407abf4047ae7f9ba8cd 2024-11-15T23:16:37,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/0e5f3681d4fc445fb7d922961bc2133e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/0e5f3681d4fc445fb7d922961bc2133e 2024-11-15T23:16:37,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/89d693e65d544a13a707cdf5e4563d86 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/89d693e65d544a13a707cdf5e4563d86 2024-11-15T23:16:37,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/c324a6a7db274453ad3f4eae149e0de6 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/c324a6a7db274453ad3f4eae149e0de6 2024-11-15T23:16:37,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/67a9ae22632e4534b5994ee7ff6c6b64 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/67a9ae22632e4534b5994ee7ff6c6b64 2024-11-15T23:16:37,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/9673c8c260644db29f5973a03b1219a4 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/9673c8c260644db29f5973a03b1219a4 2024-11-15T23:16:37,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/c9c7a10fff37447c84d1d0c1380a3a02 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/c9c7a10fff37447c84d1d0c1380a3a02 2024-11-15T23:16:37,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/571b1de536324b3ebaa70e2f20551ae0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/571b1de536324b3ebaa70e2f20551ae0 2024-11-15T23:16:37,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/2645b6aa1a5949c89129b1ad74f0b50f to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/2645b6aa1a5949c89129b1ad74f0b50f 2024-11-15T23:16:37,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/602fa765ff724d4582e351df24ac5c14 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/602fa765ff724d4582e351df24ac5c14 2024-11-15T23:16:37,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/8937567aa8344302aa4d359e50dff17f to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/8937567aa8344302aa4d359e50dff17f 2024-11-15T23:16:37,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/59059992cf55474f8f22769b93ed47a8 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/59059992cf55474f8f22769b93ed47a8 2024-11-15T23:16:37,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/f5afcf9f736b465cae6f937363da4201 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/f5afcf9f736b465cae6f937363da4201 2024-11-15T23:16:37,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/8bd98bd487b94e9a92a91a066ddb695b to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/8bd98bd487b94e9a92a91a066ddb695b 2024-11-15T23:16:37,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/a20b256acbfa429f887026e015312716 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/a20b256acbfa429f887026e015312716 2024-11-15T23:16:37,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/c8b5e5a84d1a4d1295d0fc96830a862c to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/c8b5e5a84d1a4d1295d0fc96830a862c 2024-11-15T23:16:37,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/ad0c06fb34ed46efa07064157ce5fca3 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/ad0c06fb34ed46efa07064157ce5fca3 2024-11-15T23:16:37,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/e166ae03f25943f5a1224a5e66958447 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/e166ae03f25943f5a1224a5e66958447 2024-11-15T23:16:37,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/696d0b2c764a4b859b27c4c49053880e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/696d0b2c764a4b859b27c4c49053880e 2024-11-15T23:16:37,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/b71e014b1dad4c48ad5bd648e8f782bf to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/b71e014b1dad4c48ad5bd648e8f782bf 2024-11-15T23:16:37,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/31e97a9e39c74705be91f34a4cfaa37a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/31e97a9e39c74705be91f34a4cfaa37a 2024-11-15T23:16:37,534 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/recovered.edits/502.seqid, newMaxSeqId=502, maxSeqId=1 2024-11-15T23:16:37,535 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. 2024-11-15T23:16:37,535 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for 083d01f7dba258590a0a4db0eb54177d: Waiting for close lock at 1731712596181Running coprocessor pre-close hooks at 1731712596181Disabling compacts and flushes for region at 1731712596181Disabling writes for close at 1731712596181Obtaining lock to block concurrent updates at 1731712596181Preparing flush snapshotting stores in 083d01f7dba258590a0a4db0eb54177d at 1731712596181Finished memstore snapshotting TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731712596182 (+1 ms)Flushing stores of TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d. at 1731712596183 (+1 ms)Flushing 083d01f7dba258590a0a4db0eb54177d/A: creating writer at 1731712596183Flushing 083d01f7dba258590a0a4db0eb54177d/A: appending metadata at 1731712596188 (+5 ms)Flushing 083d01f7dba258590a0a4db0eb54177d/A: closing flushed file at 1731712596189 (+1 ms)Flushing 083d01f7dba258590a0a4db0eb54177d/B: creating writer at 1731712596601 (+412 ms)Flushing 083d01f7dba258590a0a4db0eb54177d/B: appending metadata at 1731712596603 (+2 ms)Flushing 083d01f7dba258590a0a4db0eb54177d/B: closing flushed file at 1731712596603Flushing 083d01f7dba258590a0a4db0eb54177d/C: creating writer at 1731712597011 (+408 ms)Flushing 083d01f7dba258590a0a4db0eb54177d/C: appending metadata at 1731712597014 (+3 ms)Flushing 083d01f7dba258590a0a4db0eb54177d/C: closing flushed file at 1731712597014Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e84a148: reopening flushed file at 1731712597426 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cffc0b2: reopening flushed file at 1731712597433 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b0f5148: reopening flushed file at 1731712597438 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 083d01f7dba258590a0a4db0eb54177d in 1261ms, sequenceid=499, compaction requested=true at 1731712597442 (+4 ms)Writing region close event to WAL at 1731712597531 (+89 ms)Running coprocessor post-close hooks at 1731712597535 (+4 ms)Closed at 1731712597535 2024-11-15T23:16:37,537 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed 083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:37,537 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=083d01f7dba258590a0a4db0eb54177d, regionState=CLOSED 2024-11-15T23:16:37,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 because future has completed 2024-11-15T23:16:37,540 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-15T23:16:37,540 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure 083d01f7dba258590a0a4db0eb54177d, server=d6ecb9b7dda1,38353,1731712503763 in 1.5110 sec 2024-11-15T23:16:37,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-15T23:16:37,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=083d01f7dba258590a0a4db0eb54177d, UNASSIGN in 1.5150 sec 2024-11-15T23:16:37,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-15T23:16:37,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5190 sec 2024-11-15T23:16:37,544 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712597544"}]},"ts":"1731712597544"} 2024-11-15T23:16:37,545 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T23:16:37,545 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T23:16:37,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5280 sec 2024-11-15T23:16:38,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T23:16:38,176 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:38,177 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T23:16:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:16:38,180 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:16:38,182 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:16:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T23:16:38,185 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:38,188 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A, FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B, FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C, FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/recovered.edits] 2024-11-15T23:16:38,190 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/6f54b2bc319149bc893ba9271a482843 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/6f54b2bc319149bc893ba9271a482843 2024-11-15T23:16:38,191 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/ae28aeaa4bfd4d2d84cab1bd41cdef73 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/ae28aeaa4bfd4d2d84cab1bd41cdef73 2024-11-15T23:16:38,192 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/d47a8cf01da54e269eea4f2f4ca05b0a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/A/d47a8cf01da54e269eea4f2f4ca05b0a 2024-11-15T23:16:38,194 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/7a8ce3d2ff72457e97908d698132e481 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/7a8ce3d2ff72457e97908d698132e481 2024-11-15T23:16:38,194 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/7f92c1954b6b46a6bddc836dc8a8c745 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/7f92c1954b6b46a6bddc836dc8a8c745 2024-11-15T23:16:38,195 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/841542dce88343248f578efbb4393f9a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/B/841542dce88343248f578efbb4393f9a 2024-11-15T23:16:38,197 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/3f889aaef89b486ab3b17930a14d8867 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/3f889aaef89b486ab3b17930a14d8867 2024-11-15T23:16:38,198 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/57998200f83e4e7bb4e6c4895cbd2fc5 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/57998200f83e4e7bb4e6c4895cbd2fc5 2024-11-15T23:16:38,200 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/dd3e5d20a9674aef9888f8b4dddf1a62 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/C/dd3e5d20a9674aef9888f8b4dddf1a62 2024-11-15T23:16:38,202 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/recovered.edits/502.seqid to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d/recovered.edits/502.seqid 2024-11-15T23:16:38,203 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/083d01f7dba258590a0a4db0eb54177d 2024-11-15T23:16:38,203 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T23:16:38,205 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:16:38,208 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T23:16:38,209 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T23:16:38,210 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:16:38,210 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T23:16:38,210 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731712598210"}]},"ts":"9223372036854775807"} 2024-11-15T23:16:38,212 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T23:16:38,212 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 083d01f7dba258590a0a4db0eb54177d, NAME => 'TestAcidGuarantees,,1731712565739.083d01f7dba258590a0a4db0eb54177d.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T23:16:38,212 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T23:16:38,212 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731712598212"}]},"ts":"9223372036854775807"} 2024-11-15T23:16:38,214 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T23:16:38,214 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:16:38,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-11-15T23:16:38,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T23:16:38,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T23:16:38,286 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:38,299 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 252), OpenFileDescriptor=447 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=468 (was 531), ProcessCount=11 (was 11), AvailableMemoryMB=5559 (was 6348) 2024-11-15T23:16:38,308 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=468, ProcessCount=11, AvailableMemoryMB=5559 2024-11-15T23:16:38,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T23:16:38,309 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T23:16:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T23:16:38,311 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T23:16:38,311 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:38,311 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-15T23:16:38,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-15T23:16:38,312 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T23:16:38,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742163_1339 (size=1061) 2024-11-15T23:16:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-15T23:16:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-15T23:16:38,721 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444 2024-11-15T23:16:38,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742164_1340 (size=53) 2024-11-15T23:16:38,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-15T23:16:39,130 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:16:39,130 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2e9ed115eee55c880c6a2e34cdbb31c0, disabling compactions & flushes 2024-11-15T23:16:39,130 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:39,130 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:39,130 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. after waiting 0 ms 2024-11-15T23:16:39,130 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:39,130 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:39,130 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2e9ed115eee55c880c6a2e34cdbb31c0: Waiting for close lock at 1731712599130Disabling compacts and flushes for region at 1731712599130Disabling writes for close at 1731712599130Writing region close event to WAL at 1731712599130Closed at 1731712599130 2024-11-15T23:16:39,131 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T23:16:39,132 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731712599131"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731712599131"}]},"ts":"1731712599131"} 2024-11-15T23:16:39,133 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T23:16:39,134 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T23:16:39,134 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712599134"}]},"ts":"1731712599134"} 2024-11-15T23:16:39,135 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T23:16:39,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9ed115eee55c880c6a2e34cdbb31c0, ASSIGN}] 2024-11-15T23:16:39,136 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9ed115eee55c880c6a2e34cdbb31c0, ASSIGN 2024-11-15T23:16:39,137 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9ed115eee55c880c6a2e34cdbb31c0, ASSIGN; state=OFFLINE, location=d6ecb9b7dda1,38353,1731712503763; forceNewPlan=false, retain=false 2024-11-15T23:16:39,287 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=2e9ed115eee55c880c6a2e34cdbb31c0, regionState=OPENING, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:39,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9ed115eee55c880c6a2e34cdbb31c0, ASSIGN because future has completed 2024-11-15T23:16:39,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763}] 2024-11-15T23:16:39,443 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:39,443 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} 2024-11-15T23:16:39,444 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:39,444 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:16:39,444 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:39,444 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-15T23:16:39,445 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:39,446 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:16:39,446 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e9ed115eee55c880c6a2e34cdbb31c0 columnFamilyName A 2024-11-15T23:16:39,446 DEBUG [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:39,446 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] regionserver.HStore(327): Store=2e9ed115eee55c880c6a2e34cdbb31c0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:16:39,446 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:39,447 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:16:39,447 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e9ed115eee55c880c6a2e34cdbb31c0 columnFamilyName B 2024-11-15T23:16:39,447 DEBUG [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:39,448 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] regionserver.HStore(327): Store=2e9ed115eee55c880c6a2e34cdbb31c0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:16:39,448 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:39,448 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:16:39,449 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e9ed115eee55c880c6a2e34cdbb31c0 columnFamilyName C 2024-11-15T23:16:39,449 DEBUG [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:39,449 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] regionserver.HStore(327): Store=2e9ed115eee55c880c6a2e34cdbb31c0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:16:39,449 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:39,449 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:39,450 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:39,450 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:39,451 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:39,451 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:39,451 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T23:16:39,452 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:39,453 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T23:16:39,454 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 2e9ed115eee55c880c6a2e34cdbb31c0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74895733, jitterRate=0.1160333901643753}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T23:16:39,454 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:39,454 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 2e9ed115eee55c880c6a2e34cdbb31c0: Running coprocessor pre-open hook at 1731712599444Writing region info on filesystem at 1731712599444Initializing all the Stores at 1731712599445 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712599445Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712599445Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712599445Cleaning up temporary data from old regions at 1731712599451 (+6 ms)Running coprocessor post-open hooks at 1731712599454 (+3 ms)Region opened successfully at 1731712599454 2024-11-15T23:16:39,455 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., pid=87, masterSystemTime=1731712599441 2024-11-15T23:16:39,456 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:39,456 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:39,457 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=2e9ed115eee55c880c6a2e34cdbb31c0, regionState=OPEN, openSeqNum=2, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:39,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 because future has completed 2024-11-15T23:16:39,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-15T23:16:39,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 in 169 msec 2024-11-15T23:16:39,461 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-15T23:16:39,461 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9ed115eee55c880c6a2e34cdbb31c0, ASSIGN in 324 msec 2024-11-15T23:16:39,461 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T23:16:39,462 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712599461"}]},"ts":"1731712599461"} 2024-11-15T23:16:39,463 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T23:16:39,464 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T23:16:39,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1550 sec 2024-11-15T23:16:40,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-15T23:16:40,455 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:40,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d3de22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:40,456 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:40,456 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:40,457 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:40,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:40,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:40,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51fbb8a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:40,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:40,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:40,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:40,458 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:40,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e1d6a1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:40,459 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:40,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:40,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:40,460 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:40,461 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T23:16:40,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is d6ecb9b7dda1,43421,1731712502855 2024-11-15T23:16:40,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3cf4c5da 2024-11-15T23:16:40,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T23:16:40,462 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T23:16:40,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T23:16:40,463 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T23:16:40,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T23:16:40,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742165_1341 (size=1097) 2024-11-15T23:16:40,878 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-15T23:16:40,878 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-15T23:16:40,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T23:16:40,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9ed115eee55c880c6a2e34cdbb31c0, REOPEN/MOVE}] 2024-11-15T23:16:40,884 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9ed115eee55c880c6a2e34cdbb31c0, REOPEN/MOVE 2024-11-15T23:16:40,885 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=2e9ed115eee55c880c6a2e34cdbb31c0, regionState=CLOSING, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:40,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9ed115eee55c880c6a2e34cdbb31c0, REOPEN/MOVE because future has completed 2024-11-15T23:16:40,886 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T23:16:40,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763}] 2024-11-15T23:16:41,039 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:41,039 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T23:16:41,039 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 2e9ed115eee55c880c6a2e34cdbb31c0, disabling compactions & flushes 2024-11-15T23:16:41,039 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:41,039 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:41,039 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. after waiting 0 ms 2024-11-15T23:16:41,039 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:41,047 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T23:16:41,048 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:41,048 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 2e9ed115eee55c880c6a2e34cdbb31c0: Waiting for close lock at 1731712601039Running coprocessor pre-close hooks at 1731712601039Disabling compacts and flushes for region at 1731712601039Disabling writes for close at 1731712601039Writing region close event to WAL at 1731712601040 (+1 ms)Running coprocessor post-close hooks at 1731712601048 (+8 ms)Closed at 1731712601048 2024-11-15T23:16:41,048 WARN [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: 2e9ed115eee55c880c6a2e34cdbb31c0 to self. 2024-11-15T23:16:41,052 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:41,053 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=2e9ed115eee55c880c6a2e34cdbb31c0, regionState=CLOSED 2024-11-15T23:16:41,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 because future has completed 2024-11-15T23:16:41,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-15T23:16:41,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 in 170 msec 2024-11-15T23:16:41,060 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9ed115eee55c880c6a2e34cdbb31c0, REOPEN/MOVE; state=CLOSED, location=d6ecb9b7dda1,38353,1731712503763; forceNewPlan=false, retain=true 2024-11-15T23:16:41,211 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=2e9ed115eee55c880c6a2e34cdbb31c0, regionState=OPENING, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:41,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9ed115eee55c880c6a2e34cdbb31c0, REOPEN/MOVE because future has completed 2024-11-15T23:16:41,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763}] 2024-11-15T23:16:41,367 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:41,367 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} 2024-11-15T23:16:41,367 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:41,367 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:16:41,367 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:41,367 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:41,369 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:41,370 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:16:41,370 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e9ed115eee55c880c6a2e34cdbb31c0 columnFamilyName A 2024-11-15T23:16:41,371 DEBUG [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:41,372 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] regionserver.HStore(327): Store=2e9ed115eee55c880c6a2e34cdbb31c0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:16:41,372 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:41,373 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:16:41,373 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e9ed115eee55c880c6a2e34cdbb31c0 columnFamilyName B 2024-11-15T23:16:41,373 DEBUG [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:41,374 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] regionserver.HStore(327): Store=2e9ed115eee55c880c6a2e34cdbb31c0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:16:41,374 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:41,374 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:16:41,375 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e9ed115eee55c880c6a2e34cdbb31c0 columnFamilyName C 2024-11-15T23:16:41,375 DEBUG [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:41,375 INFO [StoreOpener-2e9ed115eee55c880c6a2e34cdbb31c0-1 {}] regionserver.HStore(327): Store=2e9ed115eee55c880c6a2e34cdbb31c0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:16:41,375 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:41,375 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:41,376 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:41,377 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:41,378 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:41,378 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:41,379 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T23:16:41,380 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:41,381 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened 2e9ed115eee55c880c6a2e34cdbb31c0; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71246082, jitterRate=0.06164935231208801}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T23:16:41,381 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:41,381 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for 2e9ed115eee55c880c6a2e34cdbb31c0: Running coprocessor pre-open hook at 1731712601368Writing region info on filesystem at 1731712601368Initializing all the Stores at 1731712601368Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712601368Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712601368Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712601369 (+1 ms)Cleaning up temporary data from old regions at 1731712601378 (+9 ms)Running coprocessor post-open hooks at 1731712601381 (+3 ms)Region opened successfully at 1731712601381 2024-11-15T23:16:41,382 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., pid=92, masterSystemTime=1731712601364 2024-11-15T23:16:41,384 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:41,384 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:41,384 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=2e9ed115eee55c880c6a2e34cdbb31c0, regionState=OPEN, openSeqNum=5, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:41,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 because future has completed 2024-11-15T23:16:41,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-15T23:16:41,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 in 174 msec 2024-11-15T23:16:41,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-15T23:16:41,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9ed115eee55c880c6a2e34cdbb31c0, REOPEN/MOVE in 504 msec 2024-11-15T23:16:41,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-15T23:16:41,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 509 msec 2024-11-15T23:16:41,393 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 928 msec 2024-11-15T23:16:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T23:16:41,394 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:41,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:41,395 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:41,395 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c7b7c28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:41,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:41,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:41,396 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16f5628b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,397 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:41,397 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:41,398 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:41,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:41,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:41,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16ee5a74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:41,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:41,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:41,399 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:41,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bf5c30a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,400 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:41,400 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:41,401 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58781220, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:41,402 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@106d929d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,403 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:41,403 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:41,403 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7620f6dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:41,404 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ec6ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,406 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:41,406 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:41,406 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@558fa841, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:41,407 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:41,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42de78c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,409 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:41,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:41,409 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:41,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:41,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:41,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c3c66c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:41,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:41,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:41,410 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:41,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@138f0b96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,412 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:41,412 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:41,412 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:41,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@486039d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:41,414 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:41,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e9918b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,415 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:41,415 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:41,415 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:41,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:41,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:41,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f8e009c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:41,416 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:41,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69af3fde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,417 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:41,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:41,418 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:41,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:41,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:41,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ae811c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:41,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:41,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:41,419 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4421b1ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d553196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,421 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:16:41,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:16:41,421 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:16:41,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:16:41,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:16:41,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@657f0b62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:16:41,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:16:41,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:16:41,423 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:16:41,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18cc9fff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:16:41,424 DEBUG [Thread-1569 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:41,424 DEBUG [Thread-1554 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:41,424 DEBUG [Thread-1571 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:41,424 DEBUG [Thread-1567 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:41,424 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:16:41,424 DEBUG [Thread-1565 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:41,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:41,425 DEBUG [Thread-1562 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:41,425 DEBUG [Thread-1556 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:41,425 DEBUG [Thread-1573 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-15T23:16:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:41,426 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:41,426 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:41,426 DEBUG [Thread-1562 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:41,426 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:41,426 DEBUG [Thread-1562 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:41,426 DEBUG [Thread-1573 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:41,426 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:41,427 DEBUG [Thread-1573 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:41,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T23:16:41,427 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:41,427 DEBUG [Thread-1558 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:41,427 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:41,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:41,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:41,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:41,428 DEBUG [Thread-1560 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:16:41,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:41,428 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:16:41,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:16:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:16:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:16:41,429 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:41,430 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:16:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:16:41,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:16:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:16:41,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:16:41,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:16:41,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:16:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:16:41,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:16:41,436 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:16:41,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:16:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:16:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:41,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e9ed115eee55c880c6a2e34cdbb31c0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T23:16:41,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=A 2024-11-15T23:16:41,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:41,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=B 2024-11-15T23:16:41,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:41,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=C 2024-11-15T23:16:41,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:41,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e3f5b789967046ce81196551d7a9737c_2e9ed115eee55c880c6a2e34cdbb31c0 is 50, key is test_row_0/A:col10/1731712601441/Put/seqid=0 2024-11-15T23:16:41,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731712601464, max timestamp=1731712601477Num uniques 859; 2024-11-15T23:16:41,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731712601464, max timestamp=1731712601477Num uniques 859; 2024-11-15T23:16:41,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731712601464, max timestamp=1731712601477Num uniques 859; 2024-11-15T23:16:41,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T23:16:41,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T23:16:41,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T23:16:41,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742166_1342 (size=19474) 2024-11-15T23:16:41,490 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:41,494 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e3f5b789967046ce81196551d7a9737c_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e3f5b789967046ce81196551d7a9737c_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:41,495 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/2ba3d7b2310d4c6bacdaca0c9695bc51, store: [table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:41,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/2ba3d7b2310d4c6bacdaca0c9695bc51 is 175, key is test_row_0/A:col10/1731712601441/Put/seqid=0 2024-11-15T23:16:41,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742167_1343 (size=56733) 2024-11-15T23:16:41,513 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/2ba3d7b2310d4c6bacdaca0c9695bc51 2024-11-15T23:16:41,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47444 deadline: 1731712661512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T23:16:41,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:41,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/f5fd83e2ada5426c8554eec3663bd967 is 50, key is test_row_0/B:col10/1731712601441/Put/seqid=0 2024-11-15T23:16:41,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712661533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:41,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:41,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712661533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:41,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:41,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712661535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:41,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:41,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712661535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:41,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742168_1344 (size=12001) 2024-11-15T23:16:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:41,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/f5fd83e2ada5426c8554eec3663bd967 2024-11-15T23:16:41,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T23:16:41,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:41,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:41,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:41,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/4e79944508ed42109b19c5c12c40b4c5 is 50, key is test_row_0/C:col10/1731712601441/Put/seqid=0 2024-11-15T23:16:41,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:41,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:41,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:41,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742169_1345 (size=12001) 2024-11-15T23:16:41,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/4e79944508ed42109b19c5c12c40b4c5 2024-11-15T23:16:41,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/2ba3d7b2310d4c6bacdaca0c9695bc51 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2ba3d7b2310d4c6bacdaca0c9695bc51 2024-11-15T23:16:41,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2ba3d7b2310d4c6bacdaca0c9695bc51, entries=300, sequenceid=15, filesize=55.4 K 2024-11-15T23:16:41,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/f5fd83e2ada5426c8554eec3663bd967 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/f5fd83e2ada5426c8554eec3663bd967 2024-11-15T23:16:41,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/f5fd83e2ada5426c8554eec3663bd967, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T23:16:41,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/4e79944508ed42109b19c5c12c40b4c5 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/4e79944508ed42109b19c5c12c40b4c5 2024-11-15T23:16:41,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/4e79944508ed42109b19c5c12c40b4c5, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T23:16:41,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for 2e9ed115eee55c880c6a2e34cdbb31c0 in 169ms, sequenceid=15, compaction requested=false 2024-11-15T23:16:41,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:41,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:41,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e9ed115eee55c880c6a2e34cdbb31c0 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-15T23:16:41,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=A 2024-11-15T23:16:41,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:41,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=B 2024-11-15T23:16:41,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:41,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=C 2024-11-15T23:16:41,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:41,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bd058e5963e5495a8933617e7b24a103_2e9ed115eee55c880c6a2e34cdbb31c0 is 50, key is test_row_0/A:col10/1731712601491/Put/seqid=0 2024-11-15T23:16:41,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742170_1346 (size=14594) 2024-11-15T23:16:41,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:41,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712661668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:41,685 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:41,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:41,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712661672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:41,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:41,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712661674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47444 deadline: 1731712661677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:41,691 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bd058e5963e5495a8933617e7b24a103_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bd058e5963e5495a8933617e7b24a103_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:41,692 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/d3c61e50da4a4a9f8eb12296fb3965ce, store: [table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:41,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/d3c61e50da4a4a9f8eb12296fb3965ce is 175, key is test_row_0/A:col10/1731712601491/Put/seqid=0 2024-11-15T23:16:41,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742171_1347 (size=39549) 2024-11-15T23:16:41,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712661684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:41,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:41,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T23:16:41,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:41,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:41,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:41,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:41,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T23:16:41,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712661795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:41,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712661795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:41,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47444 deadline: 1731712661795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712661795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:41,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:41,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712661806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:41,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T23:16:41,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:41,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:41,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:41,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:41,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712662015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:42,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:42,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712662015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:42,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47444 deadline: 1731712662015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:42,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712662015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:42,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712662026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:42,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T23:16:42,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:42,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:42,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:42,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T23:16:42,097 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/d3c61e50da4a4a9f8eb12296fb3965ce 2024-11-15T23:16:42,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/eded0a213c594268a0175a3c1a71389a is 50, key is test_row_0/B:col10/1731712601491/Put/seqid=0 2024-11-15T23:16:42,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742172_1348 (size=12001) 2024-11-15T23:16:42,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T23:16:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:42,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712662325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:42,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:42,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712662325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:42,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:42,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:42,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712662335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:42,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:42,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47444 deadline: 1731712662335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:42,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:42,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:42,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T23:16:42,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:42,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:42,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:42,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:42,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712662345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:42,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T23:16:42,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:42,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:42,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:42,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/eded0a213c594268a0175a3c1a71389a 2024-11-15T23:16:42,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/ba758cd37e744698adee565a2c43c195 is 50, key is test_row_0/C:col10/1731712601491/Put/seqid=0 2024-11-15T23:16:42,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742173_1349 (size=12001) 2024-11-15T23:16:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T23:16:42,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T23:16:42,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:42,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:42,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:42,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T23:16:42,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:42,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:42,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:42,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:42,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712662845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:42,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:42,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712662845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:42,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:42,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:42,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712662845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:42,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47444 deadline: 1731712662856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:42,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712662856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:42,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/ba758cd37e744698adee565a2c43c195 2024-11-15T23:16:42,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/d3c61e50da4a4a9f8eb12296fb3965ce as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/d3c61e50da4a4a9f8eb12296fb3965ce 2024-11-15T23:16:42,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/d3c61e50da4a4a9f8eb12296fb3965ce, entries=200, sequenceid=51, filesize=38.6 K 2024-11-15T23:16:42,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/eded0a213c594268a0175a3c1a71389a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/eded0a213c594268a0175a3c1a71389a 2024-11-15T23:16:42,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/eded0a213c594268a0175a3c1a71389a, entries=150, sequenceid=51, filesize=11.7 K 2024-11-15T23:16:42,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/ba758cd37e744698adee565a2c43c195 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/ba758cd37e744698adee565a2c43c195 2024-11-15T23:16:42,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T23:16:42,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:42,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:42,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:42,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/ba758cd37e744698adee565a2c43c195, entries=150, sequenceid=51, filesize=11.7 K 2024-11-15T23:16:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:42,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 2e9ed115eee55c880c6a2e34cdbb31c0 in 1306ms, sequenceid=51, compaction requested=false 2024-11-15T23:16:42,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:43,069 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T23:16:43,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T23:16:43,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:43,115 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 2e9ed115eee55c880c6a2e34cdbb31c0 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-15T23:16:43,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=A 2024-11-15T23:16:43,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:43,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=B 2024-11-15T23:16:43,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:43,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=C 2024-11-15T23:16:43,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150065f5527bb64fde8a22f7d65bd0e212_2e9ed115eee55c880c6a2e34cdbb31c0 is 50, key is test_row_0/A:col10/1731712601666/Put/seqid=0 2024-11-15T23:16:43,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742174_1350 (size=12154) 2024-11-15T23:16:43,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,163 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150065f5527bb64fde8a22f7d65bd0e212_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150065f5527bb64fde8a22f7d65bd0e212_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:43,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/6f17301d275c4b13bbd44dd4805a5ded, store: [table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:43,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/6f17301d275c4b13bbd44dd4805a5ded is 175, key is test_row_0/A:col10/1731712601666/Put/seqid=0 2024-11-15T23:16:43,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742175_1351 (size=30955) 2024-11-15T23:16:43,193 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/6f17301d275c4b13bbd44dd4805a5ded 2024-11-15T23:16:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/47b6f89d649140dbb692f126362269c5 is 50, key is test_row_0/B:col10/1731712601666/Put/seqid=0 2024-11-15T23:16:43,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742176_1352 (size=12001) 2024-11-15T23:16:43,210 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/47b6f89d649140dbb692f126362269c5 2024-11-15T23:16:43,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/73054fac56c64d2183d5ae2d637aa36f is 50, key is test_row_0/C:col10/1731712601666/Put/seqid=0 2024-11-15T23:16:43,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742177_1353 (size=12001) 2024-11-15T23:16:43,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T23:16:43,633 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/73054fac56c64d2183d5ae2d637aa36f 2024-11-15T23:16:43,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/6f17301d275c4b13bbd44dd4805a5ded as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6f17301d275c4b13bbd44dd4805a5ded 2024-11-15T23:16:43,646 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6f17301d275c4b13bbd44dd4805a5ded, entries=150, sequenceid=60, filesize=30.2 K 2024-11-15T23:16:43,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/47b6f89d649140dbb692f126362269c5 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/47b6f89d649140dbb692f126362269c5 2024-11-15T23:16:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,655 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/47b6f89d649140dbb692f126362269c5, entries=150, sequenceid=60, filesize=11.7 K 2024-11-15T23:16:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/73054fac56c64d2183d5ae2d637aa36f as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/73054fac56c64d2183d5ae2d637aa36f 2024-11-15T23:16:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,671 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/73054fac56c64d2183d5ae2d637aa36f, entries=150, sequenceid=60, filesize=11.7 K 2024-11-15T23:16:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,671 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 2e9ed115eee55c880c6a2e34cdbb31c0 in 556ms, sequenceid=60, compaction requested=true 2024-11-15T23:16:43,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:43,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-15T23:16:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-15T23:16:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-15T23:16:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2430 sec 2024-11-15T23:16:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.2520 sec 2024-11-15T23:16:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:43,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e9ed115eee55c880c6a2e34cdbb31c0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T23:16:43,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=A 2024-11-15T23:16:43,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:43,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=B 2024-11-15T23:16:43,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:43,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=C 2024-11-15T23:16:43,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a2790e94994e472a844d1db477efe32f_2e9ed115eee55c880c6a2e34cdbb31c0 is 50, key is test_row_0/A:col10/1731712603923/Put/seqid=0 2024-11-15T23:16:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742178_1354 (size=21918) 2024-11-15T23:16:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:43,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:44,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712604007, max timestamp=1731712604030Num uniques 859; 2024-11-15T23:16:44,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712604007, max timestamp=1731712604030Num uniques 859; 2024-11-15T23:16:44,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:16:44,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:16:44,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712604007, max timestamp=1731712604030Num uniques 859; 2024-11-15T23:16:44,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:16:44,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712664080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47444 deadline: 1731712664081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712664082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712664082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712664084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712664206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712664206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47444 deadline: 1731712664206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712664206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712664206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,374 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:44,377 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a2790e94994e472a844d1db477efe32f_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a2790e94994e472a844d1db477efe32f_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:44,378 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/67902745be5042bea8e71f620e45124e, store: [table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:44,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/67902745be5042bea8e71f620e45124e is 175, key is test_row_0/A:col10/1731712603923/Put/seqid=0 2024-11-15T23:16:44,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742179_1355 (size=65323) 2024-11-15T23:16:44,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712664425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712664425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712664425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47444 deadline: 1731712664426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712664435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712664745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712664745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712664745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47444 deadline: 1731712664746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712664755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:44,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:44,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:44,782 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=70, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/67902745be5042bea8e71f620e45124e 2024-11-15T23:16:44,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/79bcb8fe66994134a604f8d222f26c30 is 50, key is test_row_0/B:col10/1731712603923/Put/seqid=0 2024-11-15T23:16:44,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742180_1356 (size=12001) 2024-11-15T23:16:45,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/79bcb8fe66994134a604f8d222f26c30 2024-11-15T23:16:45,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/752f2bffb95a4da9815c5948cbdfa5a7 is 50, key is test_row_0/C:col10/1731712603923/Put/seqid=0 2024-11-15T23:16:45,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742181_1357 (size=12001) 2024-11-15T23:16:45,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712665266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:45,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712665266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:45,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712665266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:45,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47444 deadline: 1731712665266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:45,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712665275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T23:16:45,586 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:45,586 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:16:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-15T23:16:45,589 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:16:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T23:16:45,589 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:16:45,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:16:45,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/752f2bffb95a4da9815c5948cbdfa5a7 2024-11-15T23:16:45,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/67902745be5042bea8e71f620e45124e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/67902745be5042bea8e71f620e45124e 2024-11-15T23:16:45,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/67902745be5042bea8e71f620e45124e, entries=350, sequenceid=70, filesize=63.8 K 2024-11-15T23:16:45,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/79bcb8fe66994134a604f8d222f26c30 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/79bcb8fe66994134a604f8d222f26c30 2024-11-15T23:16:45,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/79bcb8fe66994134a604f8d222f26c30, entries=150, sequenceid=70, filesize=11.7 K 2024-11-15T23:16:45,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/752f2bffb95a4da9815c5948cbdfa5a7 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/752f2bffb95a4da9815c5948cbdfa5a7 2024-11-15T23:16:45,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/752f2bffb95a4da9815c5948cbdfa5a7, entries=150, sequenceid=70, filesize=11.7 K 2024-11-15T23:16:45,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 2e9ed115eee55c880c6a2e34cdbb31c0 in 1685ms, sequenceid=70, compaction requested=true 2024-11-15T23:16:45,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:45,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9ed115eee55c880c6a2e34cdbb31c0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:16:45,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:45,629 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:16:45,629 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:16:45,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9ed115eee55c880c6a2e34cdbb31c0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:16:45,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:45,631 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:16:45,631 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 192560 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:16:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9ed115eee55c880c6a2e34cdbb31c0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:16:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:45,631 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 2e9ed115eee55c880c6a2e34cdbb31c0/B is initiating minor compaction (all files) 2024-11-15T23:16:45,631 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 2e9ed115eee55c880c6a2e34cdbb31c0/A is initiating minor compaction (all files) 2024-11-15T23:16:45,631 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9ed115eee55c880c6a2e34cdbb31c0/A in TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:45,631 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9ed115eee55c880c6a2e34cdbb31c0/B in TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:45,631 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2ba3d7b2310d4c6bacdaca0c9695bc51, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/d3c61e50da4a4a9f8eb12296fb3965ce, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6f17301d275c4b13bbd44dd4805a5ded, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/67902745be5042bea8e71f620e45124e] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp, totalSize=188.0 K 2024-11-15T23:16:45,631 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/f5fd83e2ada5426c8554eec3663bd967, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/eded0a213c594268a0175a3c1a71389a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/47b6f89d649140dbb692f126362269c5, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/79bcb8fe66994134a604f8d222f26c30] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp, totalSize=46.9 K 2024-11-15T23:16:45,631 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:45,631 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. files: [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2ba3d7b2310d4c6bacdaca0c9695bc51, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/d3c61e50da4a4a9f8eb12296fb3965ce, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6f17301d275c4b13bbd44dd4805a5ded, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/67902745be5042bea8e71f620e45124e] 2024-11-15T23:16:45,631 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ba3d7b2310d4c6bacdaca0c9695bc51, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731712601433 2024-11-15T23:16:45,631 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting f5fd83e2ada5426c8554eec3663bd967, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731712601438 2024-11-15T23:16:45,631 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3c61e50da4a4a9f8eb12296fb3965ce, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731712601491 2024-11-15T23:16:45,631 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting eded0a213c594268a0175a3c1a71389a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731712601491 2024-11-15T23:16:45,632 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 47b6f89d649140dbb692f126362269c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731712601666 2024-11-15T23:16:45,632 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f17301d275c4b13bbd44dd4805a5ded, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731712601666 2024-11-15T23:16:45,632 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 79bcb8fe66994134a604f8d222f26c30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731712603923 2024-11-15T23:16:45,632 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67902745be5042bea8e71f620e45124e, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731712603913 2024-11-15T23:16:45,639 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9ed115eee55c880c6a2e34cdbb31c0#B#compaction#305 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:45,640 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/5e34effb26674286b79ef7215cf63c72 is 50, key is test_row_0/B:col10/1731712603923/Put/seqid=0 2024-11-15T23:16:45,641 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:45,653 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411155668d678068a45b3a4b59bff2dccb00f_2e9ed115eee55c880c6a2e34cdbb31c0 store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:45,656 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411155668d678068a45b3a4b59bff2dccb00f_2e9ed115eee55c880c6a2e34cdbb31c0, store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:45,657 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155668d678068a45b3a4b59bff2dccb00f_2e9ed115eee55c880c6a2e34cdbb31c0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:45,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742182_1358 (size=12139) 2024-11-15T23:16:45,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742183_1359 (size=4469) 2024-11-15T23:16:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T23:16:45,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-15T23:16:45,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:45,742 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 2e9ed115eee55c880c6a2e34cdbb31c0 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-15T23:16:45,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=A 2024-11-15T23:16:45,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:45,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=B 2024-11-15T23:16:45,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:45,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=C 2024-11-15T23:16:45,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:45,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b44f43bac6df4653afb14e694c0355a0_2e9ed115eee55c880c6a2e34cdbb31c0 is 50, key is test_row_0/A:col10/1731712604081/Put/seqid=0 2024-11-15T23:16:45,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742184_1360 (size=12154) 2024-11-15T23:16:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T23:16:46,061 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/5e34effb26674286b79ef7215cf63c72 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/5e34effb26674286b79ef7215cf63c72 2024-11-15T23:16:46,065 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2e9ed115eee55c880c6a2e34cdbb31c0/B of 2e9ed115eee55c880c6a2e34cdbb31c0 into 5e34effb26674286b79ef7215cf63c72(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:46,065 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:46,065 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., storeName=2e9ed115eee55c880c6a2e34cdbb31c0/B, priority=12, startTime=1731712605629; duration=0sec 2024-11-15T23:16:46,065 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:46,065 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9ed115eee55c880c6a2e34cdbb31c0:B 2024-11-15T23:16:46,065 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:16:46,066 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:16:46,066 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9ed115eee55c880c6a2e34cdbb31c0#A#compaction#306 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:46,066 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 2e9ed115eee55c880c6a2e34cdbb31c0/C is initiating minor compaction (all files) 2024-11-15T23:16:46,066 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9ed115eee55c880c6a2e34cdbb31c0/C in TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:46,066 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/4e79944508ed42109b19c5c12c40b4c5, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/ba758cd37e744698adee565a2c43c195, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/73054fac56c64d2183d5ae2d637aa36f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/752f2bffb95a4da9815c5948cbdfa5a7] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp, totalSize=46.9 K 2024-11-15T23:16:46,067 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e79944508ed42109b19c5c12c40b4c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731712601438 2024-11-15T23:16:46,067 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/6139e1a37048488ba53e996d422e8d5d is 175, key is test_row_0/A:col10/1731712603923/Put/seqid=0 2024-11-15T23:16:46,067 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting ba758cd37e744698adee565a2c43c195, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731712601491 2024-11-15T23:16:46,067 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 73054fac56c64d2183d5ae2d637aa36f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731712601666 2024-11-15T23:16:46,068 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 752f2bffb95a4da9815c5948cbdfa5a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731712603923 2024-11-15T23:16:46,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742185_1361 (size=31093) 2024-11-15T23:16:46,076 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/6139e1a37048488ba53e996d422e8d5d as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6139e1a37048488ba53e996d422e8d5d 2024-11-15T23:16:46,078 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9ed115eee55c880c6a2e34cdbb31c0#C#compaction#308 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:46,079 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/fb0737ab316a425f8bf35133cace8830 is 50, key is test_row_0/C:col10/1731712603923/Put/seqid=0 2024-11-15T23:16:46,082 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2e9ed115eee55c880c6a2e34cdbb31c0/A of 2e9ed115eee55c880c6a2e34cdbb31c0 into 6139e1a37048488ba53e996d422e8d5d(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:46,082 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:46,082 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., storeName=2e9ed115eee55c880c6a2e34cdbb31c0/A, priority=12, startTime=1731712605629; duration=0sec 2024-11-15T23:16:46,082 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:46,082 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9ed115eee55c880c6a2e34cdbb31c0:A 2024-11-15T23:16:46,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742186_1362 (size=12139) 2024-11-15T23:16:46,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:46,155 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b44f43bac6df4653afb14e694c0355a0_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b44f43bac6df4653afb14e694c0355a0_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:46,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/2decf746556043958c9c19461dcf2e17, store: [table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:46,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/2decf746556043958c9c19461dcf2e17 is 175, key is test_row_0/A:col10/1731712604081/Put/seqid=0 2024-11-15T23:16:46,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742187_1363 (size=30955) 2024-11-15T23:16:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T23:16:46,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:46,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47444 deadline: 1731712666295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:46,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712666295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:46,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:46,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712666301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:46,306 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in 2e9ed115eee55c880c6a2e34cdbb31c0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:16:46,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:46,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:46,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712666305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712666306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:46,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712666415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712666415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712666415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712666415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:46,544 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/fb0737ab316a425f8bf35133cace8830 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/fb0737ab316a425f8bf35133cace8830 2024-11-15T23:16:46,548 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2e9ed115eee55c880c6a2e34cdbb31c0/C of 2e9ed115eee55c880c6a2e34cdbb31c0 into fb0737ab316a425f8bf35133cace8830(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:46,548 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:46,548 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., storeName=2e9ed115eee55c880c6a2e34cdbb31c0/C, priority=12, startTime=1731712605630; duration=0sec 2024-11-15T23:16:46,549 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:46,549 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9ed115eee55c880c6a2e34cdbb31c0:C 2024-11-15T23:16:46,559 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=107, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/2decf746556043958c9c19461dcf2e17 2024-11-15T23:16:46,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/4e7a80e309b34e96ac212010f320fe21 is 50, key is test_row_0/B:col10/1731712604081/Put/seqid=0 2024-11-15T23:16:46,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742188_1364 (size=12001) 2024-11-15T23:16:46,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712666635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712666635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712666635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712666635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T23:16:46,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:46,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712666945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:46,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712666946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:46,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712666946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:46,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:46,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712666946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:46,988 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/4e7a80e309b34e96ac212010f320fe21 2024-11-15T23:16:46,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/4c525c38a3744a50971f0f8c80d116d8 is 50, key is test_row_0/C:col10/1731712604081/Put/seqid=0 2024-11-15T23:16:47,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742189_1365 (size=12001) 2024-11-15T23:16:47,403 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/4c525c38a3744a50971f0f8c80d116d8 2024-11-15T23:16:47,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/2decf746556043958c9c19461dcf2e17 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2decf746556043958c9c19461dcf2e17 2024-11-15T23:16:47,409 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2decf746556043958c9c19461dcf2e17, entries=150, sequenceid=107, filesize=30.2 K 2024-11-15T23:16:47,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/4e7a80e309b34e96ac212010f320fe21 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/4e7a80e309b34e96ac212010f320fe21 2024-11-15T23:16:47,413 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/4e7a80e309b34e96ac212010f320fe21, entries=150, sequenceid=107, filesize=11.7 K 2024-11-15T23:16:47,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/4c525c38a3744a50971f0f8c80d116d8 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/4c525c38a3744a50971f0f8c80d116d8 2024-11-15T23:16:47,417 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/4c525c38a3744a50971f0f8c80d116d8, entries=150, sequenceid=107, filesize=11.7 K 2024-11-15T23:16:47,418 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 2e9ed115eee55c880c6a2e34cdbb31c0 in 1676ms, sequenceid=107, compaction requested=false 2024-11-15T23:16:47,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:47,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:47,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-15T23:16:47,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-15T23:16:47,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-15T23:16:47,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8300 sec 2024-11-15T23:16:47,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.8360 sec 2024-11-15T23:16:47,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e9ed115eee55c880c6a2e34cdbb31c0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T23:16:47,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=A 2024-11-15T23:16:47,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:47,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=B 2024-11-15T23:16:47,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:47,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=C 2024-11-15T23:16:47,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:47,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:47,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111537018b3606bf4b978c77768c378d93ca_2e9ed115eee55c880c6a2e34cdbb31c0 is 50, key is test_row_0/A:col10/1731712607459/Put/seqid=0 2024-11-15T23:16:47,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742190_1366 (size=14594) 2024-11-15T23:16:47,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712667538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:47,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712667539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:47,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712607485, max timestamp=1731712607552Num uniques 859; 2024-11-15T23:16:47,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:16:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:47,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712607485, max timestamp=1731712607552Num uniques 859; 2024-11-15T23:16:47,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:16:47,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712607485, max timestamp=1731712607552Num uniques 859; 2024-11-15T23:16:47,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:16:47,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712667615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:47,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712667618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:47,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:47,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712667665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712667665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:47,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712667726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:47,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T23:16:47,735 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:47,735 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:16:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-15T23:16:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T23:16:47,737 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:16:47,738 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:16:47,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:16:47,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712667735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T23:16:47,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T23:16:47,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:47,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:47,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:47,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:47,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:47,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712667885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:47,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:47,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712667886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:47,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:47,908 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:47,911 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111537018b3606bf4b978c77768c378d93ca_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111537018b3606bf4b978c77768c378d93ca_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:47,912 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/6e082f7dbc5e455b9169ee93330dab47, store: [table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:47,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/6e082f7dbc5e455b9169ee93330dab47 is 175, key is test_row_0/A:col10/1731712607459/Put/seqid=0 2024-11-15T23:16:47,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742191_1367 (size=39549) 2024-11-15T23:16:47,917 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/6e082f7dbc5e455b9169ee93330dab47 2024-11-15T23:16:47,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/f7a27e5d811640ca9f184cfd1b98139a is 50, key is test_row_0/B:col10/1731712607459/Put/seqid=0 2024-11-15T23:16:47,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742192_1368 (size=12001) 2024-11-15T23:16:47,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/f7a27e5d811640ca9f184cfd1b98139a 2024-11-15T23:16:47,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/cfc8c661c7e7415e871db539ac3f3997 is 50, key is test_row_0/C:col10/1731712607459/Put/seqid=0 2024-11-15T23:16:47,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742193_1369 (size=12001) 2024-11-15T23:16:47,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/cfc8c661c7e7415e871db539ac3f3997 2024-11-15T23:16:47,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712667945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:47,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/6e082f7dbc5e455b9169ee93330dab47 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6e082f7dbc5e455b9169ee93330dab47 2024-11-15T23:16:47,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6e082f7dbc5e455b9169ee93330dab47, entries=200, sequenceid=121, filesize=38.6 K 2024-11-15T23:16:47,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/f7a27e5d811640ca9f184cfd1b98139a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/f7a27e5d811640ca9f184cfd1b98139a 2024-11-15T23:16:47,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712667960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:47,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:47,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:47,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/f7a27e5d811640ca9f184cfd1b98139a, entries=150, sequenceid=121, filesize=11.7 K 2024-11-15T23:16:47,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/cfc8c661c7e7415e871db539ac3f3997 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/cfc8c661c7e7415e871db539ac3f3997 2024-11-15T23:16:47,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/cfc8c661c7e7415e871db539ac3f3997, entries=150, sequenceid=121, filesize=11.7 K 2024-11-15T23:16:47,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 2e9ed115eee55c880c6a2e34cdbb31c0 in 519ms, sequenceid=121, compaction requested=true 2024-11-15T23:16:47,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:47,980 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:47,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9ed115eee55c880c6a2e34cdbb31c0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:16:47,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:47,981 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:47,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9ed115eee55c880c6a2e34cdbb31c0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:16:47,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:47,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9ed115eee55c880c6a2e34cdbb31c0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:16:47,984 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:47,984 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 2e9ed115eee55c880c6a2e34cdbb31c0/B is initiating minor compaction (all files) 2024-11-15T23:16:47,984 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9ed115eee55c880c6a2e34cdbb31c0/B in TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:47,984 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/5e34effb26674286b79ef7215cf63c72, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/4e7a80e309b34e96ac212010f320fe21, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/f7a27e5d811640ca9f184cfd1b98139a] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp, totalSize=35.3 K 2024-11-15T23:16:47,984 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101597 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:47,984 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 2e9ed115eee55c880c6a2e34cdbb31c0/A is initiating minor compaction (all files) 2024-11-15T23:16:47,985 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9ed115eee55c880c6a2e34cdbb31c0/A in TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:47,985 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6139e1a37048488ba53e996d422e8d5d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2decf746556043958c9c19461dcf2e17, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6e082f7dbc5e455b9169ee93330dab47] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp, totalSize=99.2 K 2024-11-15T23:16:47,985 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:47,985 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. files: [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6139e1a37048488ba53e996d422e8d5d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2decf746556043958c9c19461dcf2e17, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6e082f7dbc5e455b9169ee93330dab47] 2024-11-15T23:16:47,985 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e34effb26674286b79ef7215cf63c72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731712603923 2024-11-15T23:16:47,985 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6139e1a37048488ba53e996d422e8d5d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731712603923 2024-11-15T23:16:47,985 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e7a80e309b34e96ac212010f320fe21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731712604058 2024-11-15T23:16:47,986 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2decf746556043958c9c19461dcf2e17, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731712604058 2024-11-15T23:16:47,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:47,986 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting f7a27e5d811640ca9f184cfd1b98139a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731712606303 2024-11-15T23:16:47,986 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e082f7dbc5e455b9169ee93330dab47, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731712606292 2024-11-15T23:16:47,993 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:47,997 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9ed115eee55c880c6a2e34cdbb31c0#B#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:47,997 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/1a12b48ff2b54936bc009728cd768fd6 is 50, key is test_row_0/B:col10/1731712607459/Put/seqid=0 2024-11-15T23:16:48,001 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115b216e41f082a4faf81b2758839695f2c_2e9ed115eee55c880c6a2e34cdbb31c0 store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:48,003 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115b216e41f082a4faf81b2758839695f2c_2e9ed115eee55c880c6a2e34cdbb31c0, store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:48,003 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b216e41f082a4faf81b2758839695f2c_2e9ed115eee55c880c6a2e34cdbb31c0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:48,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742194_1370 (size=12241) 2024-11-15T23:16:48,019 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/1a12b48ff2b54936bc009728cd768fd6 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/1a12b48ff2b54936bc009728cd768fd6 2024-11-15T23:16:48,025 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9ed115eee55c880c6a2e34cdbb31c0/B of 2e9ed115eee55c880c6a2e34cdbb31c0 into 1a12b48ff2b54936bc009728cd768fd6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:48,025 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:48,025 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., storeName=2e9ed115eee55c880c6a2e34cdbb31c0/B, priority=13, startTime=1731712607981; duration=0sec 2024-11-15T23:16:48,025 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:48,025 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9ed115eee55c880c6a2e34cdbb31c0:B 2024-11-15T23:16:48,026 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:48,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742195_1371 (size=4469) 2024-11-15T23:16:48,032 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:48,032 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 2e9ed115eee55c880c6a2e34cdbb31c0/C is initiating minor compaction (all files) 2024-11-15T23:16:48,032 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9ed115eee55c880c6a2e34cdbb31c0/C in TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:48,033 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/fb0737ab316a425f8bf35133cace8830, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/4c525c38a3744a50971f0f8c80d116d8, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/cfc8c661c7e7415e871db539ac3f3997] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp, totalSize=35.3 K 2024-11-15T23:16:48,035 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9ed115eee55c880c6a2e34cdbb31c0#A#compaction#314 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:48,035 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting fb0737ab316a425f8bf35133cace8830, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731712603923 2024-11-15T23:16:48,035 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c525c38a3744a50971f0f8c80d116d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731712604058 2024-11-15T23:16:48,035 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/08d6b8d4ec3046a09c4ac3bb1a57b985 is 175, key is test_row_0/A:col10/1731712607459/Put/seqid=0 2024-11-15T23:16:48,035 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting cfc8c661c7e7415e871db539ac3f3997, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731712606303 2024-11-15T23:16:48,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T23:16:48,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:48,043 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 2e9ed115eee55c880c6a2e34cdbb31c0 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-15T23:16:48,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=A 2024-11-15T23:16:48,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:48,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=B 2024-11-15T23:16:48,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:48,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=C 2024-11-15T23:16:48,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:16:48,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T23:16:48,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742196_1372 (size=31195) 2024-11-15T23:16:48,064 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/08d6b8d4ec3046a09c4ac3bb1a57b985 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/08d6b8d4ec3046a09c4ac3bb1a57b985 2024-11-15T23:16:48,066 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9ed115eee55c880c6a2e34cdbb31c0#C#compaction#316 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:48,067 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/e461f7e5789d4837a1c3e35091f89f03 is 50, key is test_row_0/C:col10/1731712607459/Put/seqid=0 2024-11-15T23:16:48,068 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9ed115eee55c880c6a2e34cdbb31c0/A of 2e9ed115eee55c880c6a2e34cdbb31c0 into 08d6b8d4ec3046a09c4ac3bb1a57b985(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:48,068 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:48,068 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., storeName=2e9ed115eee55c880c6a2e34cdbb31c0/A, priority=13, startTime=1731712607980; duration=0sec 2024-11-15T23:16:48,068 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:48,068 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9ed115eee55c880c6a2e34cdbb31c0:A 2024-11-15T23:16:48,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b798e211f1074636aa9f82238b943b21_2e9ed115eee55c880c6a2e34cdbb31c0 is 50, key is test_row_0/A:col10/1731712607582/Put/seqid=0 2024-11-15T23:16:48,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742198_1374 (size=12304) 2024-11-15T23:16:48,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742197_1373 (size=12241) 2024-11-15T23:16:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:48,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:48,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712668228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:48,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712668238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:48,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712668255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:48,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712668275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:48,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47444 deadline: 1731712668315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:48,326 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in 2e9ed115eee55c880c6a2e34cdbb31c0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:16:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:48,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712668345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:48,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:48,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712668355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:48,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T23:16:48,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:48,493 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b798e211f1074636aa9f82238b943b21_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b798e211f1074636aa9f82238b943b21_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:48,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/12b183682544490abe2b0badbbf759ff, store: [table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:48,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/12b183682544490abe2b0badbbf759ff is 175, key is test_row_0/A:col10/1731712607582/Put/seqid=0 2024-11-15T23:16:48,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742199_1375 (size=31105) 2024-11-15T23:16:48,507 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/e461f7e5789d4837a1c3e35091f89f03 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/e461f7e5789d4837a1c3e35091f89f03 2024-11-15T23:16:48,510 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9ed115eee55c880c6a2e34cdbb31c0/C of 2e9ed115eee55c880c6a2e34cdbb31c0 into e461f7e5789d4837a1c3e35091f89f03(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:48,510 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:48,510 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., storeName=2e9ed115eee55c880c6a2e34cdbb31c0/C, priority=13, startTime=1731712607981; duration=0sec 2024-11-15T23:16:48,511 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:48,511 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9ed115eee55c880c6a2e34cdbb31c0:C 2024-11-15T23:16:48,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712668555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:48,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:48,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712668575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:48,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:48,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712668775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:48,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712668785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T23:16:48,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712668875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:48,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:48,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712668885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:48,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:48,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:48,899 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/12b183682544490abe2b0badbbf759ff 2024-11-15T23:16:48,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/2eea8a73b0624f1ea1b14d7b20df4f1b is 50, key is test_row_0/B:col10/1731712607582/Put/seqid=0 2024-11-15T23:16:48,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742200_1376 (size=12151) 2024-11-15T23:16:48,911 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/2eea8a73b0624f1ea1b14d7b20df4f1b 2024-11-15T23:16:48,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/3eb45b46ba11456e86a7157b1d8d3d07 is 50, key is test_row_0/C:col10/1731712607582/Put/seqid=0 2024-11-15T23:16:48,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742201_1377 (size=12151) 2024-11-15T23:16:49,321 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/3eb45b46ba11456e86a7157b1d8d3d07 2024-11-15T23:16:49,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/12b183682544490abe2b0badbbf759ff as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/12b183682544490abe2b0badbbf759ff 2024-11-15T23:16:49,328 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/12b183682544490abe2b0badbbf759ff, entries=150, sequenceid=157, filesize=30.4 K 2024-11-15T23:16:49,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/2eea8a73b0624f1ea1b14d7b20df4f1b as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/2eea8a73b0624f1ea1b14d7b20df4f1b 2024-11-15T23:16:49,331 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/2eea8a73b0624f1ea1b14d7b20df4f1b, entries=150, sequenceid=157, filesize=11.9 K 2024-11-15T23:16:49,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/3eb45b46ba11456e86a7157b1d8d3d07 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/3eb45b46ba11456e86a7157b1d8d3d07 2024-11-15T23:16:49,335 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/3eb45b46ba11456e86a7157b1d8d3d07, entries=150, sequenceid=157, filesize=11.9 K 2024-11-15T23:16:49,336 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 2e9ed115eee55c880c6a2e34cdbb31c0 in 1293ms, sequenceid=157, compaction requested=false 2024-11-15T23:16:49,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:49,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:49,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-15T23:16:49,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-15T23:16:49,339 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-15T23:16:49,339 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5990 sec 2024-11-15T23:16:49,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.6030 sec 2024-11-15T23:16:49,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:49,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e9ed115eee55c880c6a2e34cdbb31c0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T23:16:49,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=A 2024-11-15T23:16:49,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:49,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=B 2024-11-15T23:16:49,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:49,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=C 2024-11-15T23:16:49,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:49,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157b99c6b732f6423fbf106f928820477a_2e9ed115eee55c880c6a2e34cdbb31c0 is 50, key is test_row_0/A:col10/1731712608227/Put/seqid=0 2024-11-15T23:16:49,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742202_1378 (size=14794) 2024-11-15T23:16:49,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712669470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:49,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712669475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:49,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712669586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:49,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:49,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:49,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:49,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712669595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:49,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712669795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:49,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712669795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:49,802 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_1' in 2e9ed115eee55c880c6a2e34cdbb31c0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:16:49,802 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_1' in 2e9ed115eee55c880c6a2e34cdbb31c0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:16:49,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:49,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:49,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:49,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:49,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:49,804 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:49,807 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157b99c6b732f6423fbf106f928820477a_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157b99c6b732f6423fbf106f928820477a_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:49,807 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/0c8e1c9858d24db1ad3da8496b253a09, store: [table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:49,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/0c8e1c9858d24db1ad3da8496b253a09 is 175, key is test_row_0/A:col10/1731712608227/Put/seqid=0 2024-11-15T23:16:49,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742203_1379 (size=39749) 2024-11-15T23:16:49,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712669806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:49,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712669806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:49,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T23:16:49,896 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:49,896 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:16:49,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-15T23:16:49,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T23:16:49,898 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:16:49,898 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:16:49,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:16:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T23:16:50,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T23:16:50,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:50,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:50,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:50,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712670126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:50,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712670126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:50,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T23:16:50,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:50,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:50,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:50,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,211 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/0c8e1c9858d24db1ad3da8496b253a09 2024-11-15T23:16:50,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T23:16:50,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/9465906b13c04247a905da0aa13b7325 is 50, key is test_row_0/B:col10/1731712608227/Put/seqid=0 2024-11-15T23:16:50,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742204_1380 (size=12151) 2024-11-15T23:16:50,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T23:16:50,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:50,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:50,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:50,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T23:16:50,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:50,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:50,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:50,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T23:16:50,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/9465906b13c04247a905da0aa13b7325 2024-11-15T23:16:50,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/9e78b3a68c6e406892249ee4f821f1bc is 50, key is test_row_0/C:col10/1731712608227/Put/seqid=0 2024-11-15T23:16:50,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742205_1381 (size=12151) 2024-11-15T23:16:50,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712670645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:50,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:50,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712670645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:50,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:50,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:50,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:50,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T23:16:50,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:50,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:50,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:50,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T23:16:50,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:50,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:50,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:50,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T23:16:50,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:50,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:50,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:50,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:51,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/9e78b3a68c6e406892249ee4f821f1bc 2024-11-15T23:16:51,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/0c8e1c9858d24db1ad3da8496b253a09 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/0c8e1c9858d24db1ad3da8496b253a09 2024-11-15T23:16:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T23:16:51,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/0c8e1c9858d24db1ad3da8496b253a09, entries=200, sequenceid=171, filesize=38.8 K 2024-11-15T23:16:51,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/9465906b13c04247a905da0aa13b7325 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/9465906b13c04247a905da0aa13b7325 2024-11-15T23:16:51,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/9465906b13c04247a905da0aa13b7325, entries=150, sequenceid=171, filesize=11.9 K 2024-11-15T23:16:51,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/9e78b3a68c6e406892249ee4f821f1bc as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/9e78b3a68c6e406892249ee4f821f1bc 2024-11-15T23:16:51,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/9e78b3a68c6e406892249ee4f821f1bc, entries=150, sequenceid=171, filesize=11.9 K 2024-11-15T23:16:51,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2e9ed115eee55c880c6a2e34cdbb31c0 in 1654ms, sequenceid=171, compaction requested=true 2024-11-15T23:16:51,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:51,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9ed115eee55c880c6a2e34cdbb31c0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:16:51,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:51,045 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:51,045 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:51,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9ed115eee55c880c6a2e34cdbb31c0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:16:51,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:51,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9ed115eee55c880c6a2e34cdbb31c0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:16:51,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:51,046 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:51,046 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 2e9ed115eee55c880c6a2e34cdbb31c0/B is initiating minor compaction (all files) 2024-11-15T23:16:51,046 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9ed115eee55c880c6a2e34cdbb31c0/B in TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:51,046 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/1a12b48ff2b54936bc009728cd768fd6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/2eea8a73b0624f1ea1b14d7b20df4f1b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/9465906b13c04247a905da0aa13b7325] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp, totalSize=35.7 K 2024-11-15T23:16:51,047 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:51,047 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 2e9ed115eee55c880c6a2e34cdbb31c0/A is initiating minor compaction (all files) 2024-11-15T23:16:51,047 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9ed115eee55c880c6a2e34cdbb31c0/A in TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:51,047 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/08d6b8d4ec3046a09c4ac3bb1a57b985, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/12b183682544490abe2b0badbbf759ff, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/0c8e1c9858d24db1ad3da8496b253a09] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp, totalSize=99.7 K 2024-11-15T23:16:51,047 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:51,047 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. files: [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/08d6b8d4ec3046a09c4ac3bb1a57b985, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/12b183682544490abe2b0badbbf759ff, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/0c8e1c9858d24db1ad3da8496b253a09] 2024-11-15T23:16:51,047 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a12b48ff2b54936bc009728cd768fd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731712606303 2024-11-15T23:16:51,047 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08d6b8d4ec3046a09c4ac3bb1a57b985, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731712606303 2024-11-15T23:16:51,047 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eea8a73b0624f1ea1b14d7b20df4f1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731712607582 2024-11-15T23:16:51,047 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12b183682544490abe2b0badbbf759ff, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731712607582 2024-11-15T23:16:51,047 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 9465906b13c04247a905da0aa13b7325, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731712608227 2024-11-15T23:16:51,048 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c8e1c9858d24db1ad3da8496b253a09, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731712608221 2024-11-15T23:16:51,052 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:51,056 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9ed115eee55c880c6a2e34cdbb31c0#B#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:51,056 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411153c6dc3833a74470ebf0947a0ea35d3a2_2e9ed115eee55c880c6a2e34cdbb31c0 store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:51,056 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/5f82331e5f1e4662b87a9a6aa9b720c8 is 50, key is test_row_0/B:col10/1731712608227/Put/seqid=0 2024-11-15T23:16:51,058 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411153c6dc3833a74470ebf0947a0ea35d3a2_2e9ed115eee55c880c6a2e34cdbb31c0, store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:51,058 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153c6dc3833a74470ebf0947a0ea35d3a2_2e9ed115eee55c880c6a2e34cdbb31c0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:51,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742206_1382 (size=12493) 2024-11-15T23:16:51,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742207_1383 (size=4469) 2024-11-15T23:16:51,068 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9ed115eee55c880c6a2e34cdbb31c0#A#compaction#323 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:51,069 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/f4dd9aea1287414395b818a730d36027 is 175, key is test_row_0/A:col10/1731712608227/Put/seqid=0 2024-11-15T23:16:51,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742208_1384 (size=31447) 2024-11-15T23:16:51,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T23:16:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:51,119 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 2e9ed115eee55c880c6a2e34cdbb31c0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T23:16:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=A 2024-11-15T23:16:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=B 2024-11-15T23:16:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=C 2024-11-15T23:16:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:51,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e82bc6f79d4a4769b419dda40e5787e6_2e9ed115eee55c880c6a2e34cdbb31c0 is 50, key is test_row_0/A:col10/1731712609473/Put/seqid=0 2024-11-15T23:16:51,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742209_1385 (size=12304) 2024-11-15T23:16:51,465 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/5f82331e5f1e4662b87a9a6aa9b720c8 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/5f82331e5f1e4662b87a9a6aa9b720c8 2024-11-15T23:16:51,468 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9ed115eee55c880c6a2e34cdbb31c0/B of 2e9ed115eee55c880c6a2e34cdbb31c0 into 5f82331e5f1e4662b87a9a6aa9b720c8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:51,468 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:51,468 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., storeName=2e9ed115eee55c880c6a2e34cdbb31c0/B, priority=13, startTime=1731712611045; duration=0sec 2024-11-15T23:16:51,469 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:51,469 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9ed115eee55c880c6a2e34cdbb31c0:B 2024-11-15T23:16:51,469 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:51,470 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:51,470 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 2e9ed115eee55c880c6a2e34cdbb31c0/C is initiating minor compaction (all files) 2024-11-15T23:16:51,470 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9ed115eee55c880c6a2e34cdbb31c0/C in TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:51,470 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/e461f7e5789d4837a1c3e35091f89f03, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/3eb45b46ba11456e86a7157b1d8d3d07, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/9e78b3a68c6e406892249ee4f821f1bc] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp, totalSize=35.7 K 2024-11-15T23:16:51,470 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting e461f7e5789d4837a1c3e35091f89f03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731712606303 2024-11-15T23:16:51,470 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eb45b46ba11456e86a7157b1d8d3d07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731712607582 2024-11-15T23:16:51,471 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e78b3a68c6e406892249ee4f821f1bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731712608227 2024-11-15T23:16:51,477 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9ed115eee55c880c6a2e34cdbb31c0#C#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:51,477 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/fd1336d25ca2403aa1e39ac3a2bc29bd is 50, key is test_row_0/C:col10/1731712608227/Put/seqid=0 2024-11-15T23:16:51,482 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/f4dd9aea1287414395b818a730d36027 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/f4dd9aea1287414395b818a730d36027 2024-11-15T23:16:51,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:51,643 INFO [AsyncFSWAL-0-hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444-prefix:d6ecb9b7dda1,38353,1731712503763 {}] wal.AbstractFSWAL(1368): Slow sync cost: 156 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37491,DS-294e9834-5e94-454d-916d-1d9f671474e7,DISK]] 2024-11-15T23:16:51,643 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9ed115eee55c880c6a2e34cdbb31c0/A of 2e9ed115eee55c880c6a2e34cdbb31c0 into f4dd9aea1287414395b818a730d36027(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:51,643 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:51,643 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., storeName=2e9ed115eee55c880c6a2e34cdbb31c0/A, priority=13, startTime=1731712611045; duration=0sec 2024-11-15T23:16:51,643 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:51,643 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9ed115eee55c880c6a2e34cdbb31c0:A 2024-11-15T23:16:51,644 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e82bc6f79d4a4769b419dda40e5787e6_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e82bc6f79d4a4769b419dda40e5787e6_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:51,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742210_1386 (size=12493) 2024-11-15T23:16:51,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/d4b83d2a11de44e6be0710d69a62b498, store: [table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:51,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/d4b83d2a11de44e6be0710d69a62b498 is 175, key is test_row_0/A:col10/1731712609473/Put/seqid=0 2024-11-15T23:16:51,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742211_1387 (size=31105) 2024-11-15T23:16:51,660 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/d4b83d2a11de44e6be0710d69a62b498 2024-11-15T23:16:51,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/a696875705d54754936b26f1ee4c2173 is 50, key is test_row_0/B:col10/1731712609473/Put/seqid=0 2024-11-15T23:16:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:51,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:51,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742212_1388 (size=12151) 2024-11-15T23:16:51,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712671694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:51,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712671703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:51,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712671816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:51,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712671816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:51,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712671816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:51,821 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_1' in 2e9ed115eee55c880c6a2e34cdbb31c0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:16:51,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:51,821 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_1' in 2e9ed115eee55c880c6a2e34cdbb31c0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:16:51,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:51,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712671816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:52,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:52,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712672025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:52,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:52,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:52,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712672025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:52,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:52,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:52,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:52,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:52,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T23:16:52,048 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/fd1336d25ca2403aa1e39ac3a2bc29bd as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/fd1336d25ca2403aa1e39ac3a2bc29bd 2024-11-15T23:16:52,051 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9ed115eee55c880c6a2e34cdbb31c0/C of 2e9ed115eee55c880c6a2e34cdbb31c0 into fd1336d25ca2403aa1e39ac3a2bc29bd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:52,052 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:52,052 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., storeName=2e9ed115eee55c880c6a2e34cdbb31c0/C, priority=13, startTime=1731712611045; duration=0sec 2024-11-15T23:16:52,052 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:52,052 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9ed115eee55c880c6a2e34cdbb31c0:C 2024-11-15T23:16:52,073 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/a696875705d54754936b26f1ee4c2173 2024-11-15T23:16:52,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/b0c5f4137fa54f50af70c93d796b92f3 is 50, key is test_row_0/C:col10/1731712609473/Put/seqid=0 2024-11-15T23:16:52,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742213_1389 (size=12151) 2024-11-15T23:16:52,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712672335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:52,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712672345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:52,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:52,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:52,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:52,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47444 deadline: 1731712672355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:52,361 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in 2e9ed115eee55c880c6a2e34cdbb31c0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8279 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:16:52,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:52,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:52,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:52,484 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/b0c5f4137fa54f50af70c93d796b92f3 2024-11-15T23:16:52,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/d4b83d2a11de44e6be0710d69a62b498 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/d4b83d2a11de44e6be0710d69a62b498 2024-11-15T23:16:52,497 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/d4b83d2a11de44e6be0710d69a62b498, entries=150, sequenceid=191, filesize=30.4 K 2024-11-15T23:16:52,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/a696875705d54754936b26f1ee4c2173 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/a696875705d54754936b26f1ee4c2173 2024-11-15T23:16:52,506 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/a696875705d54754936b26f1ee4c2173, entries=150, sequenceid=191, filesize=11.9 K 2024-11-15T23:16:52,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/b0c5f4137fa54f50af70c93d796b92f3 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/b0c5f4137fa54f50af70c93d796b92f3 2024-11-15T23:16:52,510 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/b0c5f4137fa54f50af70c93d796b92f3, entries=150, sequenceid=191, filesize=11.9 K 2024-11-15T23:16:52,512 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2e9ed115eee55c880c6a2e34cdbb31c0 in 1392ms, sequenceid=191, compaction requested=false 2024-11-15T23:16:52,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:52,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:52,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-15T23:16:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-15T23:16:52,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-15T23:16:52,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6170 sec 2024-11-15T23:16:52,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.6220 sec 2024-11-15T23:16:52,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:52,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e9ed115eee55c880c6a2e34cdbb31c0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T23:16:52,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=A 2024-11-15T23:16:52,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:52,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=B 2024-11-15T23:16:52,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:52,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=C 2024-11-15T23:16:52,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:52,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ddf747fac8124f529f03ded11865634a_2e9ed115eee55c880c6a2e34cdbb31c0 is 50, key is test_row_0/A:col10/1731712611680/Put/seqid=0 2024-11-15T23:16:52,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742214_1390 (size=14794) 2024-11-15T23:16:52,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712672918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:52,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712672920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:53,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:53,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712673025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:53,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:53,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712673035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:53,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:53,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:53,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:53,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:53,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712673235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:53,262 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:53,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:53,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712673255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:53,265 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ddf747fac8124f529f03ded11865634a_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ddf747fac8124f529f03ded11865634a_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:53,266 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/73ff4b6a1fd04eed8093c5ad7d17d8b6, store: [table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:53,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/73ff4b6a1fd04eed8093c5ad7d17d8b6 is 175, key is test_row_0/A:col10/1731712611680/Put/seqid=0 2024-11-15T23:16:53,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742215_1391 (size=39749) 2024-11-15T23:16:53,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712673555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:53,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:53,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:53,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:53,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:53,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712673576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:53,673 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/73ff4b6a1fd04eed8093c5ad7d17d8b6 2024-11-15T23:16:53,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/b9cef502165040008a3f223c1ceaa5ec is 50, key is test_row_0/B:col10/1731712611680/Put/seqid=0 2024-11-15T23:16:53,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742216_1392 (size=12151) 2024-11-15T23:16:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T23:16:54,056 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:54,056 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:16:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-15T23:16:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T23:16:54,057 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:16:54,058 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:16:54,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:16:54,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712674076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:54,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:54,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/b9cef502165040008a3f223c1ceaa5ec 2024-11-15T23:16:54,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/ea80aaad67994b6c9cae3084bb81e081 is 50, key is test_row_0/C:col10/1731712611680/Put/seqid=0 2024-11-15T23:16:54,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742217_1393 (size=12151) 2024-11-15T23:16:54,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712674085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:54,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:54,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:54,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T23:16:54,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T23:16:54,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:54,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:54,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:54,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:54,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:54,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T23:16:54,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:54,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:54,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:54,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:54,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T23:16:54,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/ea80aaad67994b6c9cae3084bb81e081 2024-11-15T23:16:54,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/73ff4b6a1fd04eed8093c5ad7d17d8b6 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/73ff4b6a1fd04eed8093c5ad7d17d8b6 2024-11-15T23:16:54,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/73ff4b6a1fd04eed8093c5ad7d17d8b6, entries=200, sequenceid=207, filesize=38.8 K 2024-11-15T23:16:54,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/b9cef502165040008a3f223c1ceaa5ec as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/b9cef502165040008a3f223c1ceaa5ec 2024-11-15T23:16:54,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/b9cef502165040008a3f223c1ceaa5ec, entries=150, sequenceid=207, filesize=11.9 K 2024-11-15T23:16:54,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/ea80aaad67994b6c9cae3084bb81e081 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/ea80aaad67994b6c9cae3084bb81e081 2024-11-15T23:16:54,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/ea80aaad67994b6c9cae3084bb81e081, entries=150, sequenceid=207, filesize=11.9 K 2024-11-15T23:16:54,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 2e9ed115eee55c880c6a2e34cdbb31c0 in 1660ms, sequenceid=207, compaction requested=true 2024-11-15T23:16:54,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:54,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9ed115eee55c880c6a2e34cdbb31c0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:16:54,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:54,510 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:54,510 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:54,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9ed115eee55c880c6a2e34cdbb31c0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:16:54,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:54,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9ed115eee55c880c6a2e34cdbb31c0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:16:54,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:54,510 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:54,511 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 2e9ed115eee55c880c6a2e34cdbb31c0/B is initiating minor compaction (all files) 2024-11-15T23:16:54,511 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9ed115eee55c880c6a2e34cdbb31c0/B in TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:54,511 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/5f82331e5f1e4662b87a9a6aa9b720c8, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/a696875705d54754936b26f1ee4c2173, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/b9cef502165040008a3f223c1ceaa5ec] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp, totalSize=35.9 K 2024-11-15T23:16:54,511 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:54,511 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f82331e5f1e4662b87a9a6aa9b720c8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731712608227 2024-11-15T23:16:54,511 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting a696875705d54754936b26f1ee4c2173, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731712609415 2024-11-15T23:16:54,512 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting b9cef502165040008a3f223c1ceaa5ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731712611680 2024-11-15T23:16:54,513 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 2e9ed115eee55c880c6a2e34cdbb31c0/A is initiating minor compaction (all files) 2024-11-15T23:16:54,513 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9ed115eee55c880c6a2e34cdbb31c0/A in TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:54,513 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/f4dd9aea1287414395b818a730d36027, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/d4b83d2a11de44e6be0710d69a62b498, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/73ff4b6a1fd04eed8093c5ad7d17d8b6] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp, totalSize=99.9 K 2024-11-15T23:16:54,513 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:54,513 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. files: [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/f4dd9aea1287414395b818a730d36027, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/d4b83d2a11de44e6be0710d69a62b498, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/73ff4b6a1fd04eed8093c5ad7d17d8b6] 2024-11-15T23:16:54,513 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4dd9aea1287414395b818a730d36027, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731712608227 2024-11-15T23:16:54,514 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4b83d2a11de44e6be0710d69a62b498, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731712609415 2024-11-15T23:16:54,514 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73ff4b6a1fd04eed8093c5ad7d17d8b6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731712611680 2024-11-15T23:16:54,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T23:16:54,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:54,517 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 2e9ed115eee55c880c6a2e34cdbb31c0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T23:16:54,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=A 2024-11-15T23:16:54,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:54,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=B 2024-11-15T23:16:54,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:54,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=C 2024-11-15T23:16:54,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:54,519 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9ed115eee55c880c6a2e34cdbb31c0#B#compaction#332 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:54,519 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/207f076bb89e476fbd56f48a729aa75c is 50, key is test_row_0/B:col10/1731712611680/Put/seqid=0 2024-11-15T23:16:54,523 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:54,526 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115296412fc89f14f5987d447c9f96f484b_2e9ed115eee55c880c6a2e34cdbb31c0 store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:54,528 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115296412fc89f14f5987d447c9f96f484b_2e9ed115eee55c880c6a2e34cdbb31c0, store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:54,528 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115296412fc89f14f5987d447c9f96f484b_2e9ed115eee55c880c6a2e34cdbb31c0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:54,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111538e64df05f21492897771c0d1df58bad_2e9ed115eee55c880c6a2e34cdbb31c0 is 50, key is test_row_0/A:col10/1731712612887/Put/seqid=0 2024-11-15T23:16:54,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742218_1394 (size=12595) 2024-11-15T23:16:54,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742219_1395 (size=4469) 2024-11-15T23:16:54,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742220_1396 (size=12304) 2024-11-15T23:16:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T23:16:54,943 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/207f076bb89e476fbd56f48a729aa75c as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/207f076bb89e476fbd56f48a729aa75c 2024-11-15T23:16:54,947 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9ed115eee55c880c6a2e34cdbb31c0/B of 2e9ed115eee55c880c6a2e34cdbb31c0 into 207f076bb89e476fbd56f48a729aa75c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:54,947 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:54,947 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., storeName=2e9ed115eee55c880c6a2e34cdbb31c0/B, priority=13, startTime=1731712614510; duration=0sec 2024-11-15T23:16:54,947 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:54,947 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9ed115eee55c880c6a2e34cdbb31c0:B 2024-11-15T23:16:54,947 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:54,948 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:54,948 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 2e9ed115eee55c880c6a2e34cdbb31c0/C is initiating minor compaction (all files) 2024-11-15T23:16:54,948 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9ed115eee55c880c6a2e34cdbb31c0/C in TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:54,949 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/fd1336d25ca2403aa1e39ac3a2bc29bd, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/b0c5f4137fa54f50af70c93d796b92f3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/ea80aaad67994b6c9cae3084bb81e081] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp, totalSize=35.9 K 2024-11-15T23:16:54,949 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting fd1336d25ca2403aa1e39ac3a2bc29bd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731712608227 2024-11-15T23:16:54,949 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting b0c5f4137fa54f50af70c93d796b92f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731712609415 2024-11-15T23:16:54,949 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting ea80aaad67994b6c9cae3084bb81e081, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731712611680 2024-11-15T23:16:54,950 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9ed115eee55c880c6a2e34cdbb31c0#A#compaction#333 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:54,950 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/746becaf084d46beb54021062000f6c6 is 175, key is test_row_0/A:col10/1731712611680/Put/seqid=0 2024-11-15T23:16:54,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742221_1397 (size=31549) 2024-11-15T23:16:54,957 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9ed115eee55c880c6a2e34cdbb31c0#C#compaction#335 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:54,957 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/4c9fdc07b4b544eb88e921bc250a8245 is 50, key is test_row_0/C:col10/1731712611680/Put/seqid=0 2024-11-15T23:16:54,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:54,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742222_1398 (size=12595) 2024-11-15T23:16:54,961 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111538e64df05f21492897771c0d1df58bad_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111538e64df05f21492897771c0d1df58bad_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:54,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/ca2720b454b44e828708d2b4dd9f9ff0, store: [table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:54,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/ca2720b454b44e828708d2b4dd9f9ff0 is 175, key is test_row_0/A:col10/1731712612887/Put/seqid=0 2024-11-15T23:16:54,964 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/4c9fdc07b4b544eb88e921bc250a8245 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/4c9fdc07b4b544eb88e921bc250a8245 2024-11-15T23:16:54,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742223_1399 (size=31105) 2024-11-15T23:16:54,968 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9ed115eee55c880c6a2e34cdbb31c0/C of 2e9ed115eee55c880c6a2e34cdbb31c0 into 4c9fdc07b4b544eb88e921bc250a8245(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:54,968 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:54,968 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., storeName=2e9ed115eee55c880c6a2e34cdbb31c0/C, priority=13, startTime=1731712614510; duration=0sec 2024-11-15T23:16:54,968 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:54,969 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9ed115eee55c880c6a2e34cdbb31c0:C 2024-11-15T23:16:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:55,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:55,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712675137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:55,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:55,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712675144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T23:16:55,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712675255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:55,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:55,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:55,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712675265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:55,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:55,359 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/746becaf084d46beb54021062000f6c6 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/746becaf084d46beb54021062000f6c6 2024-11-15T23:16:55,363 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9ed115eee55c880c6a2e34cdbb31c0/A of 2e9ed115eee55c880c6a2e34cdbb31c0 into 746becaf084d46beb54021062000f6c6(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:55,363 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:55,363 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., storeName=2e9ed115eee55c880c6a2e34cdbb31c0/A, priority=13, startTime=1731712614510; duration=0sec 2024-11-15T23:16:55,363 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:55,363 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9ed115eee55c880c6a2e34cdbb31c0:A 2024-11-15T23:16:55,366 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/ca2720b454b44e828708d2b4dd9f9ff0 2024-11-15T23:16:55,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/67bbb2bc07d94c24a4e2115ab6b608ba is 50, key is test_row_0/B:col10/1731712612887/Put/seqid=0 2024-11-15T23:16:55,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742224_1400 (size=12151) 2024-11-15T23:16:55,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712675465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:55,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712675475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:55,774 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/67bbb2bc07d94c24a4e2115ab6b608ba 2024-11-15T23:16:55,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/95ce107556f445c3b50c1900ac3615c9 is 50, key is test_row_0/C:col10/1731712612887/Put/seqid=0 2024-11-15T23:16:55,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742225_1401 (size=12151) 2024-11-15T23:16:55,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712675775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:55,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:55,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712675785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:55,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731712675825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:55,832 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_1' in 2e9ed115eee55c880c6a2e34cdbb31c0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:16:55,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:55,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47424 deadline: 1731712675845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:55,852 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_1' in 2e9ed115eee55c880c6a2e34cdbb31c0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:16:55,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:55,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:56,184 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/95ce107556f445c3b50c1900ac3615c9 2024-11-15T23:16:56,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/ca2720b454b44e828708d2b4dd9f9ff0 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/ca2720b454b44e828708d2b4dd9f9ff0 2024-11-15T23:16:56,191 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/ca2720b454b44e828708d2b4dd9f9ff0, entries=150, sequenceid=226, filesize=30.4 K 2024-11-15T23:16:56,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/67bbb2bc07d94c24a4e2115ab6b608ba as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/67bbb2bc07d94c24a4e2115ab6b608ba 2024-11-15T23:16:56,195 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/67bbb2bc07d94c24a4e2115ab6b608ba, entries=150, sequenceid=226, filesize=11.9 K 2024-11-15T23:16:56,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/95ce107556f445c3b50c1900ac3615c9 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/95ce107556f445c3b50c1900ac3615c9 2024-11-15T23:16:56,200 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/95ce107556f445c3b50c1900ac3615c9, entries=150, sequenceid=226, filesize=11.9 K 2024-11-15T23:16:56,201 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 2e9ed115eee55c880c6a2e34cdbb31c0 in 1685ms, sequenceid=226, compaction requested=false 2024-11-15T23:16:56,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:56,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:56,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-15T23:16:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-15T23:16:56,204 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-15T23:16:56,204 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1440 sec 2024-11-15T23:16:56,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.1480 sec 2024-11-15T23:16:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T23:16:56,206 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:16:56,206 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:16:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-15T23:16:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T23:16:56,208 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:16:56,209 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:16:56,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:16:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:56,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e9ed115eee55c880c6a2e34cdbb31c0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T23:16:56,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=A 2024-11-15T23:16:56,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:56,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=B 2024-11-15T23:16:56,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:56,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=C 2024-11-15T23:16:56,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:56,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115815fbfaaa1ee408fab1c4a3a70bc1fa5_2e9ed115eee55c880c6a2e34cdbb31c0 is 50, key is test_row_0/A:col10/1731712616299/Put/seqid=0 2024-11-15T23:16:56,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742226_1402 (size=14794) 2024-11-15T23:16:56,309 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:56,312 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115815fbfaaa1ee408fab1c4a3a70bc1fa5_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115815fbfaaa1ee408fab1c4a3a70bc1fa5_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:56,313 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/2aec8e185f534f1cbca6d3988fc02346, store: [table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:56,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/2aec8e185f534f1cbca6d3988fc02346 is 175, key is test_row_0/A:col10/1731712616299/Put/seqid=0 2024-11-15T23:16:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T23:16:56,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742227_1403 (size=39749) 2024-11-15T23:16:56,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T23:16:56,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:56,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:56,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:56,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:56,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:56,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:56,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712676367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:56,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:56,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712676370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:56,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:56,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:56,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:56,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712676485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:56,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:56,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:56,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712676485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:56,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:56,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T23:16:56,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:56,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:56,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:56,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:56,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T23:16:56,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T23:16:56,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:56,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:56,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:56,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:56,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:56,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712676695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:56,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712676695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:56,717 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/2aec8e185f534f1cbca6d3988fc02346 2024-11-15T23:16:56,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/84524878c6654dec9510430f947eae14 is 50, key is test_row_0/B:col10/1731712616299/Put/seqid=0 2024-11-15T23:16:56,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742228_1404 (size=12151) 2024-11-15T23:16:56,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/84524878c6654dec9510430f947eae14 2024-11-15T23:16:56,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/edf8622e6bee450e9c5ccad5e9763e84 is 50, key is test_row_0/C:col10/1731712616299/Put/seqid=0 2024-11-15T23:16:56,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742229_1405 (size=12151) 2024-11-15T23:16:56,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T23:16:56,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:56,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:56,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:56,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:56,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:56,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T23:16:56,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T23:16:56,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:56,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:56,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:56,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:56,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:57,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712677005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:57,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712677005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:57,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T23:16:57,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:57,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:57,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:57,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:57,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:16:57,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/edf8622e6bee450e9c5ccad5e9763e84 2024-11-15T23:16:57,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/2aec8e185f534f1cbca6d3988fc02346 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2aec8e185f534f1cbca6d3988fc02346 2024-11-15T23:16:57,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2aec8e185f534f1cbca6d3988fc02346, entries=200, sequenceid=243, filesize=38.8 K 2024-11-15T23:16:57,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/84524878c6654dec9510430f947eae14 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/84524878c6654dec9510430f947eae14 2024-11-15T23:16:57,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/84524878c6654dec9510430f947eae14, entries=150, sequenceid=243, filesize=11.9 K 2024-11-15T23:16:57,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/edf8622e6bee450e9c5ccad5e9763e84 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/edf8622e6bee450e9c5ccad5e9763e84 2024-11-15T23:16:57,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/edf8622e6bee450e9c5ccad5e9763e84, entries=150, sequenceid=243, filesize=11.9 K 2024-11-15T23:16:57,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 2e9ed115eee55c880c6a2e34cdbb31c0 in 855ms, sequenceid=243, compaction requested=true 2024-11-15T23:16:57,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:57,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9ed115eee55c880c6a2e34cdbb31c0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:16:57,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:57,156 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:57,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9ed115eee55c880c6a2e34cdbb31c0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:16:57,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:57,156 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:57,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9ed115eee55c880c6a2e34cdbb31c0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:16:57,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:57,157 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:57,157 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:57,157 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 2e9ed115eee55c880c6a2e34cdbb31c0/B is initiating minor compaction (all files) 2024-11-15T23:16:57,157 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 2e9ed115eee55c880c6a2e34cdbb31c0/A is initiating minor compaction (all files) 2024-11-15T23:16:57,157 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9ed115eee55c880c6a2e34cdbb31c0/B in TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:57,157 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9ed115eee55c880c6a2e34cdbb31c0/A in TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:57,157 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/746becaf084d46beb54021062000f6c6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/ca2720b454b44e828708d2b4dd9f9ff0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2aec8e185f534f1cbca6d3988fc02346] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp, totalSize=100.0 K 2024-11-15T23:16:57,157 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/207f076bb89e476fbd56f48a729aa75c, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/67bbb2bc07d94c24a4e2115ab6b608ba, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/84524878c6654dec9510430f947eae14] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp, totalSize=36.0 K 2024-11-15T23:16:57,157 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:57,157 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. files: [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/746becaf084d46beb54021062000f6c6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/ca2720b454b44e828708d2b4dd9f9ff0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2aec8e185f534f1cbca6d3988fc02346] 2024-11-15T23:16:57,157 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 207f076bb89e476fbd56f48a729aa75c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731712611680 2024-11-15T23:16:57,158 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 746becaf084d46beb54021062000f6c6, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731712611680 2024-11-15T23:16:57,158 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 67bbb2bc07d94c24a4e2115ab6b608ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731712612887 2024-11-15T23:16:57,158 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca2720b454b44e828708d2b4dd9f9ff0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731712612887 2024-11-15T23:16:57,158 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 84524878c6654dec9510430f947eae14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731712615134 2024-11-15T23:16:57,158 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2aec8e185f534f1cbca6d3988fc02346, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731712615117 2024-11-15T23:16:57,165 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9ed115eee55c880c6a2e34cdbb31c0#B#compaction#341 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:57,165 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/2471e5319a3343478c70e9573cff51fc is 50, key is test_row_0/B:col10/1731712616299/Put/seqid=0 2024-11-15T23:16:57,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742230_1406 (size=12697) 2024-11-15T23:16:57,176 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:57,178 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111595fc493ea9fb439782e422113c00a1df_2e9ed115eee55c880c6a2e34cdbb31c0 store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:57,180 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111595fc493ea9fb439782e422113c00a1df_2e9ed115eee55c880c6a2e34cdbb31c0, store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:57,181 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111595fc493ea9fb439782e422113c00a1df_2e9ed115eee55c880c6a2e34cdbb31c0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:57,182 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/2471e5319a3343478c70e9573cff51fc as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/2471e5319a3343478c70e9573cff51fc 2024-11-15T23:16:57,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742231_1407 (size=4469) 2024-11-15T23:16:57,186 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9ed115eee55c880c6a2e34cdbb31c0/B of 2e9ed115eee55c880c6a2e34cdbb31c0 into 2471e5319a3343478c70e9573cff51fc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:57,186 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:57,186 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., storeName=2e9ed115eee55c880c6a2e34cdbb31c0/B, priority=13, startTime=1731712617156; duration=0sec 2024-11-15T23:16:57,186 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:57,186 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9ed115eee55c880c6a2e34cdbb31c0:B 2024-11-15T23:16:57,186 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:57,186 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:57,186 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 2e9ed115eee55c880c6a2e34cdbb31c0/C is initiating minor compaction (all files) 2024-11-15T23:16:57,186 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9ed115eee55c880c6a2e34cdbb31c0/C in TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:57,187 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/4c9fdc07b4b544eb88e921bc250a8245, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/95ce107556f445c3b50c1900ac3615c9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/edf8622e6bee450e9c5ccad5e9763e84] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp, totalSize=36.0 K 2024-11-15T23:16:57,187 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c9fdc07b4b544eb88e921bc250a8245, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731712611680 2024-11-15T23:16:57,187 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 95ce107556f445c3b50c1900ac3615c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731712612887 2024-11-15T23:16:57,187 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting edf8622e6bee450e9c5ccad5e9763e84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731712615134 2024-11-15T23:16:57,193 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9ed115eee55c880c6a2e34cdbb31c0#C#compaction#343 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:57,193 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/c2c9d85e3c6847159a8e76ea4598202a is 50, key is test_row_0/C:col10/1731712616299/Put/seqid=0 2024-11-15T23:16:57,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742232_1408 (size=12697) 2024-11-15T23:16:57,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T23:16:57,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:57,283 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 2e9ed115eee55c880c6a2e34cdbb31c0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T23:16:57,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=A 2024-11-15T23:16:57,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:57,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=B 2024-11-15T23:16:57,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:57,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=C 2024-11-15T23:16:57,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:57,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158d90e863969141f9b566c68d384ddeb9_2e9ed115eee55c880c6a2e34cdbb31c0 is 50, key is test_row_0/A:col10/1731712616369/Put/seqid=0 2024-11-15T23:16:57,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742233_1409 (size=12454) 2024-11-15T23:16:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T23:16:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:57,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:16:57,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712677569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:57,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712677572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:57,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:57,586 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9ed115eee55c880c6a2e34cdbb31c0#A#compaction#342 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:57,587 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/7cd27bcee5a54fb79f619d99cfe1b2fc is 175, key is test_row_0/A:col10/1731712616299/Put/seqid=0 2024-11-15T23:16:57,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742234_1410 (size=31651) 2024-11-15T23:16:57,601 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/c2c9d85e3c6847159a8e76ea4598202a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/c2c9d85e3c6847159a8e76ea4598202a 2024-11-15T23:16:57,604 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9ed115eee55c880c6a2e34cdbb31c0/C of 2e9ed115eee55c880c6a2e34cdbb31c0 into c2c9d85e3c6847159a8e76ea4598202a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:57,604 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:57,604 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., storeName=2e9ed115eee55c880c6a2e34cdbb31c0/C, priority=13, startTime=1731712617156; duration=0sec 2024-11-15T23:16:57,605 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:57,605 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9ed115eee55c880c6a2e34cdbb31c0:C 2024-11-15T23:16:57,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:57,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712677685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:57,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712677685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:57,695 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158d90e863969141f9b566c68d384ddeb9_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158d90e863969141f9b566c68d384ddeb9_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:57,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/20a9fe6409f3401099a19078eecd7235, store: [table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:57,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/20a9fe6409f3401099a19078eecd7235 is 175, key is test_row_0/A:col10/1731712616369/Put/seqid=0 2024-11-15T23:16:57,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742235_1411 (size=31255) 2024-11-15T23:16:57,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712677896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:57,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712677896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:57,997 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/7cd27bcee5a54fb79f619d99cfe1b2fc as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/7cd27bcee5a54fb79f619d99cfe1b2fc 2024-11-15T23:16:58,006 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9ed115eee55c880c6a2e34cdbb31c0/A of 2e9ed115eee55c880c6a2e34cdbb31c0 into 7cd27bcee5a54fb79f619d99cfe1b2fc(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:16:58,006 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:58,006 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., storeName=2e9ed115eee55c880c6a2e34cdbb31c0/A, priority=13, startTime=1731712617155; duration=0sec 2024-11-15T23:16:58,006 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:58,006 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9ed115eee55c880c6a2e34cdbb31c0:A 2024-11-15T23:16:58,099 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/20a9fe6409f3401099a19078eecd7235 2024-11-15T23:16:58,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/ef776f57622b4f0ca370336a3484bcfd is 50, key is test_row_0/B:col10/1731712616369/Put/seqid=0 2024-11-15T23:16:58,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742236_1412 (size=12301) 2024-11-15T23:16:58,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712678215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:58,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712678215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T23:16:58,509 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/ef776f57622b4f0ca370336a3484bcfd 2024-11-15T23:16:58,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/e870cb012db64b7580b75b365dd7db7a is 50, key is test_row_0/C:col10/1731712616369/Put/seqid=0 2024-11-15T23:16:58,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742237_1413 (size=12301) 2024-11-15T23:16:58,534 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/e870cb012db64b7580b75b365dd7db7a 2024-11-15T23:16:58,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/20a9fe6409f3401099a19078eecd7235 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/20a9fe6409f3401099a19078eecd7235 2024-11-15T23:16:58,550 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/20a9fe6409f3401099a19078eecd7235, entries=150, sequenceid=262, filesize=30.5 K 2024-11-15T23:16:58,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/ef776f57622b4f0ca370336a3484bcfd as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/ef776f57622b4f0ca370336a3484bcfd 2024-11-15T23:16:58,556 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/ef776f57622b4f0ca370336a3484bcfd, entries=150, sequenceid=262, filesize=12.0 K 2024-11-15T23:16:58,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/e870cb012db64b7580b75b365dd7db7a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/e870cb012db64b7580b75b365dd7db7a 2024-11-15T23:16:58,562 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/e870cb012db64b7580b75b365dd7db7a, entries=150, sequenceid=262, filesize=12.0 K 2024-11-15T23:16:58,564 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2e9ed115eee55c880c6a2e34cdbb31c0 in 1281ms, sequenceid=262, compaction requested=false 2024-11-15T23:16:58,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:58,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:58,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-15T23:16:58,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-15T23:16:58,567 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-15T23:16:58,567 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3570 sec 2024-11-15T23:16:58,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.3610 sec 2024-11-15T23:16:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:58,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e9ed115eee55c880c6a2e34cdbb31c0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T23:16:58,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=A 2024-11-15T23:16:58,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:58,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=B 2024-11-15T23:16:58,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:58,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=C 2024-11-15T23:16:58,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:58,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c17e9ffd63f54db0bf392b40b2f00382_2e9ed115eee55c880c6a2e34cdbb31c0 is 50, key is test_row_0/A:col10/1731712617571/Put/seqid=0 2024-11-15T23:16:58,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742238_1414 (size=14994) 2024-11-15T23:16:58,742 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:16:58,745 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c17e9ffd63f54db0bf392b40b2f00382_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c17e9ffd63f54db0bf392b40b2f00382_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:58,746 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/aa7df9faefbf479fa08a692db4b0c13a, store: [table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:58,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/aa7df9faefbf479fa08a692db4b0c13a is 175, key is test_row_0/A:col10/1731712617571/Put/seqid=0 2024-11-15T23:16:58,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742239_1415 (size=39949) 2024-11-15T23:16:58,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712678805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:58,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:58,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:58,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:58,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712678805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:58,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:58,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:58,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:58,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712678925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:58,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712678925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:59,149 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/aa7df9faefbf479fa08a692db4b0c13a 2024-11-15T23:16:59,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712679145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:59,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712679145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:59,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/6aef769de76c4716a9e9a5f2061a4767 is 50, key is test_row_0/B:col10/1731712617571/Put/seqid=0 2024-11-15T23:16:59,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742240_1416 (size=12301) 2024-11-15T23:16:59,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/6aef769de76c4716a9e9a5f2061a4767 2024-11-15T23:16:59,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/8f1a7614580144ad874f81094fe1051e is 50, key is test_row_0/C:col10/1731712617571/Put/seqid=0 2024-11-15T23:16:59,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742241_1417 (size=12301) 2024-11-15T23:16:59,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712679465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:59,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:16:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712679465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:16:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:16:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:16:59,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/8f1a7614580144ad874f81094fe1051e 2024-11-15T23:16:59,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/aa7df9faefbf479fa08a692db4b0c13a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/aa7df9faefbf479fa08a692db4b0c13a 2024-11-15T23:16:59,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/aa7df9faefbf479fa08a692db4b0c13a, entries=200, sequenceid=279, filesize=39.0 K 2024-11-15T23:16:59,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/6aef769de76c4716a9e9a5f2061a4767 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/6aef769de76c4716a9e9a5f2061a4767 2024-11-15T23:16:59,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/6aef769de76c4716a9e9a5f2061a4767, entries=150, sequenceid=279, filesize=12.0 K 2024-11-15T23:16:59,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/8f1a7614580144ad874f81094fe1051e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/8f1a7614580144ad874f81094fe1051e 2024-11-15T23:16:59,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/8f1a7614580144ad874f81094fe1051e, entries=150, sequenceid=279, filesize=12.0 K 2024-11-15T23:16:59,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 2e9ed115eee55c880c6a2e34cdbb31c0 in 859ms, sequenceid=279, compaction requested=true 2024-11-15T23:16:59,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:16:59,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9ed115eee55c880c6a2e34cdbb31c0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:16:59,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:59,591 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:59,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9ed115eee55c880c6a2e34cdbb31c0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:16:59,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:16:59,591 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:16:59,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9ed115eee55c880c6a2e34cdbb31c0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:16:59,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:16:59,591 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:59,591 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:16:59,591 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 2e9ed115eee55c880c6a2e34cdbb31c0/B is initiating minor compaction (all files) 2024-11-15T23:16:59,591 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 2e9ed115eee55c880c6a2e34cdbb31c0/A is initiating minor compaction (all files) 2024-11-15T23:16:59,592 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9ed115eee55c880c6a2e34cdbb31c0/B in TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:59,592 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9ed115eee55c880c6a2e34cdbb31c0/A in TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:59,592 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/2471e5319a3343478c70e9573cff51fc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/ef776f57622b4f0ca370336a3484bcfd, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/6aef769de76c4716a9e9a5f2061a4767] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp, totalSize=36.4 K 2024-11-15T23:16:59,592 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/7cd27bcee5a54fb79f619d99cfe1b2fc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/20a9fe6409f3401099a19078eecd7235, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/aa7df9faefbf479fa08a692db4b0c13a] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp, totalSize=100.4 K 2024-11-15T23:16:59,592 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:16:59,592 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. files: [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/7cd27bcee5a54fb79f619d99cfe1b2fc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/20a9fe6409f3401099a19078eecd7235, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/aa7df9faefbf479fa08a692db4b0c13a] 2024-11-15T23:16:59,592 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 2471e5319a3343478c70e9573cff51fc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731712615134 2024-11-15T23:16:59,592 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cd27bcee5a54fb79f619d99cfe1b2fc, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731712615134 2024-11-15T23:16:59,592 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting ef776f57622b4f0ca370336a3484bcfd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731712616359 2024-11-15T23:16:59,592 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20a9fe6409f3401099a19078eecd7235, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731712616359 2024-11-15T23:16:59,593 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 6aef769de76c4716a9e9a5f2061a4767, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731712617567 2024-11-15T23:16:59,593 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa7df9faefbf479fa08a692db4b0c13a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731712617543 2024-11-15T23:16:59,599 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9ed115eee55c880c6a2e34cdbb31c0#B#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:16:59,599 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/10b4c9d0caf74a09b24399836528bf29 is 50, key is test_row_0/B:col10/1731712617571/Put/seqid=0 2024-11-15T23:16:59,605 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:59,607 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115bda0064f51444fedb09664ec4149f9b7_2e9ed115eee55c880c6a2e34cdbb31c0 store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:59,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742242_1418 (size=12949) 2024-11-15T23:16:59,610 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115bda0064f51444fedb09664ec4149f9b7_2e9ed115eee55c880c6a2e34cdbb31c0, store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:59,610 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bda0064f51444fedb09664ec4149f9b7_2e9ed115eee55c880c6a2e34cdbb31c0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:16:59,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742243_1419 (size=4469) 2024-11-15T23:16:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:16:59,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e9ed115eee55c880c6a2e34cdbb31c0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T23:16:59,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=A 2024-11-15T23:16:59,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:59,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=B 2024-11-15T23:16:59,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:59,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=C 2024-11-15T23:16:59,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:16:59,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c8b624772e0e44e0b49a9e960ce7afbd_2e9ed115eee55c880c6a2e34cdbb31c0 is 50, key is test_row_0/A:col10/1731712618769/Put/seqid=0 2024-11-15T23:16:59,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742244_1420 (size=14994) 2024-11-15T23:17:00,011 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/10b4c9d0caf74a09b24399836528bf29 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/10b4c9d0caf74a09b24399836528bf29 2024-11-15T23:17:00,015 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9ed115eee55c880c6a2e34cdbb31c0/B of 2e9ed115eee55c880c6a2e34cdbb31c0 into 10b4c9d0caf74a09b24399836528bf29(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:00,015 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:17:00,015 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., storeName=2e9ed115eee55c880c6a2e34cdbb31c0/B, priority=13, startTime=1731712619591; duration=0sec 2024-11-15T23:17:00,015 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:00,015 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9ed115eee55c880c6a2e34cdbb31c0:B 2024-11-15T23:17:00,015 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:00,016 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:00,016 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 2e9ed115eee55c880c6a2e34cdbb31c0/C is initiating minor compaction (all files) 2024-11-15T23:17:00,016 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9ed115eee55c880c6a2e34cdbb31c0/C in TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:00,016 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/c2c9d85e3c6847159a8e76ea4598202a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/e870cb012db64b7580b75b365dd7db7a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/8f1a7614580144ad874f81094fe1051e] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp, totalSize=36.4 K 2024-11-15T23:17:00,016 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting c2c9d85e3c6847159a8e76ea4598202a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731712615134 2024-11-15T23:17:00,016 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting e870cb012db64b7580b75b365dd7db7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731712616359 2024-11-15T23:17:00,017 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f1a7614580144ad874f81094fe1051e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731712617567 2024-11-15T23:17:00,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712680009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:00,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:00,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:00,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:00,022 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9ed115eee55c880c6a2e34cdbb31c0#C#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:00,023 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/0189edca291f42f98472999ffcdc7ae8 is 50, key is test_row_0/C:col10/1731712617571/Put/seqid=0 2024-11-15T23:17:00,023 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9ed115eee55c880c6a2e34cdbb31c0#A#compaction#351 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:00,023 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/9bfc666e2f824928acb27e2fdda145fb is 175, key is test_row_0/A:col10/1731712617571/Put/seqid=0 2024-11-15T23:17:00,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742246_1422 (size=31903) 2024-11-15T23:17:00,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742245_1421 (size=12949) 2024-11-15T23:17:00,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712680021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:00,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:00,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:00,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712680125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:00,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712680135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:00,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:00,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:00,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:00,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712680335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:00,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:00,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:00,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:00,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712680355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:00,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:00,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:00,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T23:17:00,365 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:00,366 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:17:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-15T23:17:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T23:17:00,367 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:17:00,368 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:17:00,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:17:00,395 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:00,401 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c8b624772e0e44e0b49a9e960ce7afbd_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c8b624772e0e44e0b49a9e960ce7afbd_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:00,402 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/b56c7c0be4ce4efdb9735f11c42d47a5, store: [table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:17:00,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/b56c7c0be4ce4efdb9735f11c42d47a5 is 175, key is test_row_0/A:col10/1731712618769/Put/seqid=0 2024-11-15T23:17:00,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742247_1423 (size=39949) 2024-11-15T23:17:00,430 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/9bfc666e2f824928acb27e2fdda145fb as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/9bfc666e2f824928acb27e2fdda145fb 2024-11-15T23:17:00,435 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/0189edca291f42f98472999ffcdc7ae8 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/0189edca291f42f98472999ffcdc7ae8 2024-11-15T23:17:00,436 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9ed115eee55c880c6a2e34cdbb31c0/A of 2e9ed115eee55c880c6a2e34cdbb31c0 into 9bfc666e2f824928acb27e2fdda145fb(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:00,436 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:17:00,436 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., storeName=2e9ed115eee55c880c6a2e34cdbb31c0/A, priority=13, startTime=1731712619591; duration=0sec 2024-11-15T23:17:00,436 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:00,436 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9ed115eee55c880c6a2e34cdbb31c0:A 2024-11-15T23:17:00,439 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9ed115eee55c880c6a2e34cdbb31c0/C of 2e9ed115eee55c880c6a2e34cdbb31c0 into 0189edca291f42f98472999ffcdc7ae8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:00,440 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:17:00,440 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., storeName=2e9ed115eee55c880c6a2e34cdbb31c0/C, priority=13, startTime=1731712619591; duration=0sec 2024-11-15T23:17:00,440 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:00,440 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9ed115eee55c880c6a2e34cdbb31c0:C 2024-11-15T23:17:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T23:17:00,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T23:17:00,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:00,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:17:00,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:00,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:00,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:00,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:00,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712680655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:00,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T23:17:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:17:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:00,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:00,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:00,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712680675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T23:17:00,805 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/b56c7c0be4ce4efdb9735f11c42d47a5 2024-11-15T23:17:00,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/212ddb4b344c41dfafc9788fe70063da is 50, key is test_row_0/B:col10/1731712618769/Put/seqid=0 2024-11-15T23:17:00,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742248_1424 (size=12301) 2024-11-15T23:17:00,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T23:17:00,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:00,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:17:00,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:00,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:00,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:00,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T23:17:00,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:00,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:17:00,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:00,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:00,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:00,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T23:17:01,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T23:17:01,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:01,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:17:01,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:01,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:01,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:01,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47420 deadline: 1731712681165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:01,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:01,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:01,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:01,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47446 deadline: 1731712681195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:01,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:01,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:01,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:01,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/212ddb4b344c41dfafc9788fe70063da 2024-11-15T23:17:01,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/d46df57e5514420e995122eaff36184f is 50, key is test_row_0/C:col10/1731712618769/Put/seqid=0 2024-11-15T23:17:01,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742249_1425 (size=12301) 2024-11-15T23:17:01,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T23:17:01,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:01,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:17:01,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:01,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:01,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:01,431 INFO [Thread-1569 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1569. 2024-11-15T23:17:01,431 DEBUG [Thread-1569 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:01,431 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:01,432 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:01,432 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:01,433 INFO [Thread-1565 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1565. 2024-11-15T23:17:01,433 DEBUG [Thread-1565 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:01,433 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:01,433 INFO [Thread-1567 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1567. 2024-11-15T23:17:01,433 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:01,433 DEBUG [Thread-1567 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:01,433 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:01,433 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:01,433 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:01,433 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:01,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T23:17:01,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:01,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:17:01,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:01,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:01,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:01,441 INFO [Thread-1573 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1573. 2024-11-15T23:17:01,441 DEBUG [Thread-1573 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:01,441 DEBUG [Thread-1573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:01,441 DEBUG [Thread-1573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:01,441 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:01,442 INFO [Thread-1571 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1571. 2024-11-15T23:17:01,442 DEBUG [Thread-1571 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:01,442 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:01,442 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:01,442 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T23:17:01,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T23:17:01,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:01,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:17:01,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:01,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:01,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:01,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/d46df57e5514420e995122eaff36184f 2024-11-15T23:17:01,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/b56c7c0be4ce4efdb9735f11c42d47a5 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/b56c7c0be4ce4efdb9735f11c42d47a5 2024-11-15T23:17:01,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/b56c7c0be4ce4efdb9735f11c42d47a5, entries=200, sequenceid=298, filesize=39.0 K 2024-11-15T23:17:01,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/212ddb4b344c41dfafc9788fe70063da as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/212ddb4b344c41dfafc9788fe70063da 2024-11-15T23:17:01,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/212ddb4b344c41dfafc9788fe70063da, entries=150, sequenceid=298, filesize=12.0 K 2024-11-15T23:17:01,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/d46df57e5514420e995122eaff36184f as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/d46df57e5514420e995122eaff36184f 2024-11-15T23:17:01,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/d46df57e5514420e995122eaff36184f, entries=150, sequenceid=298, filesize=12.0 K 2024-11-15T23:17:01,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 2e9ed115eee55c880c6a2e34cdbb31c0 in 1665ms, sequenceid=298, compaction requested=false 2024-11-15T23:17:01,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:17:01,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T23:17:01,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:01,744 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 2e9ed115eee55c880c6a2e34cdbb31c0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T23:17:01,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=A 2024-11-15T23:17:01,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:01,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=B 2024-11-15T23:17:01,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:01,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=C 2024-11-15T23:17:01,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:01,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111553373645acab489cb4b70c72e3c71e26_2e9ed115eee55c880c6a2e34cdbb31c0 is 50, key is test_row_0/A:col10/1731712620002/Put/seqid=0 2024-11-15T23:17:01,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742250_1426 (size=12454) 2024-11-15T23:17:01,968 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T23:17:02,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:02,168 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111553373645acab489cb4b70c72e3c71e26_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111553373645acab489cb4b70c72e3c71e26_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:02,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/73c36801cbda40fb8ac47936e7048ab6, store: [table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:17:02,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/73c36801cbda40fb8ac47936e7048ab6 is 175, key is test_row_0/A:col10/1731712620002/Put/seqid=0 2024-11-15T23:17:02,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742251_1427 (size=31255) 2024-11-15T23:17:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:02,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. as already flushing 2024-11-15T23:17:02,177 INFO [Thread-1556 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1556. 2024-11-15T23:17:02,177 DEBUG [Thread-1556 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:02,177 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:02,177 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:02,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:02,218 INFO [Thread-1560 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1560. 2024-11-15T23:17:02,218 DEBUG [Thread-1560 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:02,219 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:02,219 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:02,219 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:02,396 INFO [Thread-1558 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1558. 2024-11-15T23:17:02,396 DEBUG [Thread-1558 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:02,396 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:02,397 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:02,397 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:02,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T23:17:02,575 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/73c36801cbda40fb8ac47936e7048ab6 2024-11-15T23:17:02,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/d52b289760034d01bea381afc9d6b836 is 50, key is test_row_0/B:col10/1731712620002/Put/seqid=0 2024-11-15T23:17:02,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742252_1428 (size=12301) 2024-11-15T23:17:03,017 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/d52b289760034d01bea381afc9d6b836 2024-11-15T23:17:03,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/c491ebe5cfcd416da3fd9c0df891a9bc is 50, key is test_row_0/C:col10/1731712620002/Put/seqid=0 2024-11-15T23:17:03,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742253_1429 (size=12301) 2024-11-15T23:17:03,436 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/c491ebe5cfcd416da3fd9c0df891a9bc 2024-11-15T23:17:03,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/73c36801cbda40fb8ac47936e7048ab6 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/73c36801cbda40fb8ac47936e7048ab6 2024-11-15T23:17:03,453 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/73c36801cbda40fb8ac47936e7048ab6, entries=150, sequenceid=314, filesize=30.5 K 2024-11-15T23:17:03,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/d52b289760034d01bea381afc9d6b836 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/d52b289760034d01bea381afc9d6b836 2024-11-15T23:17:03,457 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/d52b289760034d01bea381afc9d6b836, entries=150, sequenceid=314, filesize=12.0 K 2024-11-15T23:17:03,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/c491ebe5cfcd416da3fd9c0df891a9bc as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/c491ebe5cfcd416da3fd9c0df891a9bc 2024-11-15T23:17:03,462 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/c491ebe5cfcd416da3fd9c0df891a9bc, entries=150, sequenceid=314, filesize=12.0 K 2024-11-15T23:17:03,463 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=20.13 KB/20610 for 2e9ed115eee55c880c6a2e34cdbb31c0 in 1720ms, sequenceid=314, compaction requested=true 2024-11-15T23:17:03,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 2e9ed115eee55c880c6a2e34cdbb31c0: 2024-11-15T23:17:03,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:03,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-15T23:17:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-15T23:17:03,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-15T23:17:03,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0960 sec 2024-11-15T23:17:03,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 3.1010 sec 2024-11-15T23:17:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T23:17:04,536 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:05,860 INFO [Thread-1554 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1554. 2024-11-15T23:17:05,860 DEBUG [Thread-1554 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:05,860 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:05,860 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:05,861 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:05,927 INFO [Thread-1562 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1562. 2024-11-15T23:17:05,927 DEBUG [Thread-1562 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:05,927 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:05,927 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:05,928 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:05,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T23:17:05,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-15T23:17:05,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-15T23:17:05,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-15T23:17:05,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-15T23:17:05,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-15T23:17:05,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T23:17:05,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T23:17:05,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1188 2024-11-15T23:17:05,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3559 rows 2024-11-15T23:17:05,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1202 2024-11-15T23:17:05,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3592 rows 2024-11-15T23:17:05,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1194 2024-11-15T23:17:05,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3580 rows 2024-11-15T23:17:05,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1189 2024-11-15T23:17:05,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3564 rows 2024-11-15T23:17:05,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1195 2024-11-15T23:17:05,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3585 rows 2024-11-15T23:17:05,928 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T23:17:05,928 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T23:17:05,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:05,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:05,929 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:05,929 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T23:17:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T23:17:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T23:17:05,933 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712625933"}]},"ts":"1731712625933"} 2024-11-15T23:17:05,934 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T23:17:05,935 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T23:17:05,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T23:17:05,936 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9ed115eee55c880c6a2e34cdbb31c0, UNASSIGN}] 2024-11-15T23:17:05,937 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9ed115eee55c880c6a2e34cdbb31c0, UNASSIGN 2024-11-15T23:17:05,937 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=2e9ed115eee55c880c6a2e34cdbb31c0, regionState=CLOSING, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:05,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9ed115eee55c880c6a2e34cdbb31c0, UNASSIGN because future has completed 2024-11-15T23:17:05,939 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T23:17:05,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763}] 2024-11-15T23:17:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T23:17:06,091 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:06,091 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T23:17:06,091 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing 2e9ed115eee55c880c6a2e34cdbb31c0, disabling compactions & flushes 2024-11-15T23:17:06,091 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:06,092 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:06,092 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. after waiting 0 ms 2024-11-15T23:17:06,092 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:06,092 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing 2e9ed115eee55c880c6a2e34cdbb31c0 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-15T23:17:06,092 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=A 2024-11-15T23:17:06,092 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:06,092 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=B 2024-11-15T23:17:06,092 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:06,093 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2e9ed115eee55c880c6a2e34cdbb31c0, store=C 2024-11-15T23:17:06,093 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:06,102 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f018b2a8991c4d999eb609213caec43c_2e9ed115eee55c880c6a2e34cdbb31c0 is 50, key is test_row_1/A:col10/1731712625926/Put/seqid=0 2024-11-15T23:17:06,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742254_1430 (size=9914) 2024-11-15T23:17:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T23:17:06,509 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:06,519 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f018b2a8991c4d999eb609213caec43c_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f018b2a8991c4d999eb609213caec43c_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:06,520 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/0a0fb2700187406d95b709a51dddca74, store: [table=TestAcidGuarantees family=A region=2e9ed115eee55c880c6a2e34cdbb31c0] 2024-11-15T23:17:06,521 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/0a0fb2700187406d95b709a51dddca74 is 175, key is test_row_1/A:col10/1731712625926/Put/seqid=0 2024-11-15T23:17:06,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742255_1431 (size=22561) 2024-11-15T23:17:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T23:17:06,756 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2ba3d7b2310d4c6bacdaca0c9695bc51, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/d3c61e50da4a4a9f8eb12296fb3965ce, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6f17301d275c4b13bbd44dd4805a5ded, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/67902745be5042bea8e71f620e45124e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6139e1a37048488ba53e996d422e8d5d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2decf746556043958c9c19461dcf2e17, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6e082f7dbc5e455b9169ee93330dab47, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/08d6b8d4ec3046a09c4ac3bb1a57b985, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/12b183682544490abe2b0badbbf759ff, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/0c8e1c9858d24db1ad3da8496b253a09, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/f4dd9aea1287414395b818a730d36027, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/d4b83d2a11de44e6be0710d69a62b498, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/73ff4b6a1fd04eed8093c5ad7d17d8b6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/746becaf084d46beb54021062000f6c6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/ca2720b454b44e828708d2b4dd9f9ff0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2aec8e185f534f1cbca6d3988fc02346, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/7cd27bcee5a54fb79f619d99cfe1b2fc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/20a9fe6409f3401099a19078eecd7235, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/aa7df9faefbf479fa08a692db4b0c13a] to archive 2024-11-15T23:17:06,757 WARN [regionserver/d6ecb9b7dda1:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-11-15T23:17:06,758 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:17:06,760 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2ba3d7b2310d4c6bacdaca0c9695bc51 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2ba3d7b2310d4c6bacdaca0c9695bc51 2024-11-15T23:17:06,762 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/d3c61e50da4a4a9f8eb12296fb3965ce to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/d3c61e50da4a4a9f8eb12296fb3965ce 2024-11-15T23:17:06,764 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6f17301d275c4b13bbd44dd4805a5ded to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6f17301d275c4b13bbd44dd4805a5ded 2024-11-15T23:17:06,765 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/67902745be5042bea8e71f620e45124e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/67902745be5042bea8e71f620e45124e 2024-11-15T23:17:06,767 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6139e1a37048488ba53e996d422e8d5d to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6139e1a37048488ba53e996d422e8d5d 2024-11-15T23:17:06,768 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2decf746556043958c9c19461dcf2e17 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2decf746556043958c9c19461dcf2e17 2024-11-15T23:17:06,770 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6e082f7dbc5e455b9169ee93330dab47 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/6e082f7dbc5e455b9169ee93330dab47 2024-11-15T23:17:06,771 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/08d6b8d4ec3046a09c4ac3bb1a57b985 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/08d6b8d4ec3046a09c4ac3bb1a57b985 2024-11-15T23:17:06,772 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/12b183682544490abe2b0badbbf759ff to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/12b183682544490abe2b0badbbf759ff 2024-11-15T23:17:06,773 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/0c8e1c9858d24db1ad3da8496b253a09 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/0c8e1c9858d24db1ad3da8496b253a09 2024-11-15T23:17:06,774 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/f4dd9aea1287414395b818a730d36027 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/f4dd9aea1287414395b818a730d36027 2024-11-15T23:17:06,775 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/d4b83d2a11de44e6be0710d69a62b498 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/d4b83d2a11de44e6be0710d69a62b498 2024-11-15T23:17:06,776 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/73ff4b6a1fd04eed8093c5ad7d17d8b6 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/73ff4b6a1fd04eed8093c5ad7d17d8b6 2024-11-15T23:17:06,778 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/746becaf084d46beb54021062000f6c6 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/746becaf084d46beb54021062000f6c6 2024-11-15T23:17:06,779 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/ca2720b454b44e828708d2b4dd9f9ff0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/ca2720b454b44e828708d2b4dd9f9ff0 2024-11-15T23:17:06,779 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2aec8e185f534f1cbca6d3988fc02346 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/2aec8e185f534f1cbca6d3988fc02346 2024-11-15T23:17:06,780 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/7cd27bcee5a54fb79f619d99cfe1b2fc to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/7cd27bcee5a54fb79f619d99cfe1b2fc 2024-11-15T23:17:06,781 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/20a9fe6409f3401099a19078eecd7235 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/20a9fe6409f3401099a19078eecd7235 2024-11-15T23:17:06,782 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/aa7df9faefbf479fa08a692db4b0c13a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/aa7df9faefbf479fa08a692db4b0c13a 2024-11-15T23:17:06,784 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/f5fd83e2ada5426c8554eec3663bd967, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/eded0a213c594268a0175a3c1a71389a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/47b6f89d649140dbb692f126362269c5, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/5e34effb26674286b79ef7215cf63c72, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/79bcb8fe66994134a604f8d222f26c30, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/4e7a80e309b34e96ac212010f320fe21, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/1a12b48ff2b54936bc009728cd768fd6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/f7a27e5d811640ca9f184cfd1b98139a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/2eea8a73b0624f1ea1b14d7b20df4f1b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/5f82331e5f1e4662b87a9a6aa9b720c8, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/9465906b13c04247a905da0aa13b7325, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/a696875705d54754936b26f1ee4c2173, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/207f076bb89e476fbd56f48a729aa75c, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/b9cef502165040008a3f223c1ceaa5ec, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/67bbb2bc07d94c24a4e2115ab6b608ba, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/2471e5319a3343478c70e9573cff51fc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/84524878c6654dec9510430f947eae14, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/ef776f57622b4f0ca370336a3484bcfd, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/6aef769de76c4716a9e9a5f2061a4767] to archive 2024-11-15T23:17:06,785 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:17:06,786 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/f5fd83e2ada5426c8554eec3663bd967 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/f5fd83e2ada5426c8554eec3663bd967 2024-11-15T23:17:06,787 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/eded0a213c594268a0175a3c1a71389a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/eded0a213c594268a0175a3c1a71389a 2024-11-15T23:17:06,788 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/47b6f89d649140dbb692f126362269c5 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/47b6f89d649140dbb692f126362269c5 2024-11-15T23:17:06,789 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/5e34effb26674286b79ef7215cf63c72 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/5e34effb26674286b79ef7215cf63c72 2024-11-15T23:17:06,790 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/79bcb8fe66994134a604f8d222f26c30 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/79bcb8fe66994134a604f8d222f26c30 2024-11-15T23:17:06,791 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/4e7a80e309b34e96ac212010f320fe21 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/4e7a80e309b34e96ac212010f320fe21 2024-11-15T23:17:06,792 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/1a12b48ff2b54936bc009728cd768fd6 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/1a12b48ff2b54936bc009728cd768fd6 2024-11-15T23:17:06,793 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/f7a27e5d811640ca9f184cfd1b98139a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/f7a27e5d811640ca9f184cfd1b98139a 2024-11-15T23:17:06,794 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/2eea8a73b0624f1ea1b14d7b20df4f1b to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/2eea8a73b0624f1ea1b14d7b20df4f1b 2024-11-15T23:17:06,795 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/5f82331e5f1e4662b87a9a6aa9b720c8 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/5f82331e5f1e4662b87a9a6aa9b720c8 2024-11-15T23:17:06,796 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/9465906b13c04247a905da0aa13b7325 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/9465906b13c04247a905da0aa13b7325 2024-11-15T23:17:06,797 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/a696875705d54754936b26f1ee4c2173 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/a696875705d54754936b26f1ee4c2173 2024-11-15T23:17:06,798 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/207f076bb89e476fbd56f48a729aa75c to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/207f076bb89e476fbd56f48a729aa75c 2024-11-15T23:17:06,798 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/b9cef502165040008a3f223c1ceaa5ec to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/b9cef502165040008a3f223c1ceaa5ec 2024-11-15T23:17:06,799 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/67bbb2bc07d94c24a4e2115ab6b608ba to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/67bbb2bc07d94c24a4e2115ab6b608ba 2024-11-15T23:17:06,800 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/2471e5319a3343478c70e9573cff51fc to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/2471e5319a3343478c70e9573cff51fc 2024-11-15T23:17:06,801 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/84524878c6654dec9510430f947eae14 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/84524878c6654dec9510430f947eae14 2024-11-15T23:17:06,802 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/ef776f57622b4f0ca370336a3484bcfd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/ef776f57622b4f0ca370336a3484bcfd 2024-11-15T23:17:06,803 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/6aef769de76c4716a9e9a5f2061a4767 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/6aef769de76c4716a9e9a5f2061a4767 2024-11-15T23:17:06,805 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/4e79944508ed42109b19c5c12c40b4c5, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/ba758cd37e744698adee565a2c43c195, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/73054fac56c64d2183d5ae2d637aa36f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/fb0737ab316a425f8bf35133cace8830, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/752f2bffb95a4da9815c5948cbdfa5a7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/4c525c38a3744a50971f0f8c80d116d8, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/e461f7e5789d4837a1c3e35091f89f03, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/cfc8c661c7e7415e871db539ac3f3997, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/3eb45b46ba11456e86a7157b1d8d3d07, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/fd1336d25ca2403aa1e39ac3a2bc29bd, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/9e78b3a68c6e406892249ee4f821f1bc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/b0c5f4137fa54f50af70c93d796b92f3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/4c9fdc07b4b544eb88e921bc250a8245, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/ea80aaad67994b6c9cae3084bb81e081, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/95ce107556f445c3b50c1900ac3615c9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/c2c9d85e3c6847159a8e76ea4598202a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/edf8622e6bee450e9c5ccad5e9763e84, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/e870cb012db64b7580b75b365dd7db7a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/8f1a7614580144ad874f81094fe1051e] to archive 2024-11-15T23:17:06,806 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:17:06,807 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/4e79944508ed42109b19c5c12c40b4c5 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/4e79944508ed42109b19c5c12c40b4c5 2024-11-15T23:17:06,808 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/ba758cd37e744698adee565a2c43c195 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/ba758cd37e744698adee565a2c43c195 2024-11-15T23:17:06,809 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/73054fac56c64d2183d5ae2d637aa36f to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/73054fac56c64d2183d5ae2d637aa36f 2024-11-15T23:17:06,810 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/fb0737ab316a425f8bf35133cace8830 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/fb0737ab316a425f8bf35133cace8830 2024-11-15T23:17:06,811 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/752f2bffb95a4da9815c5948cbdfa5a7 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/752f2bffb95a4da9815c5948cbdfa5a7 2024-11-15T23:17:06,811 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/4c525c38a3744a50971f0f8c80d116d8 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/4c525c38a3744a50971f0f8c80d116d8 2024-11-15T23:17:06,812 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/e461f7e5789d4837a1c3e35091f89f03 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/e461f7e5789d4837a1c3e35091f89f03 2024-11-15T23:17:06,813 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/cfc8c661c7e7415e871db539ac3f3997 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/cfc8c661c7e7415e871db539ac3f3997 2024-11-15T23:17:06,814 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/3eb45b46ba11456e86a7157b1d8d3d07 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/3eb45b46ba11456e86a7157b1d8d3d07 2024-11-15T23:17:06,814 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/fd1336d25ca2403aa1e39ac3a2bc29bd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/fd1336d25ca2403aa1e39ac3a2bc29bd 2024-11-15T23:17:06,815 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/9e78b3a68c6e406892249ee4f821f1bc to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/9e78b3a68c6e406892249ee4f821f1bc 2024-11-15T23:17:06,816 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/b0c5f4137fa54f50af70c93d796b92f3 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/b0c5f4137fa54f50af70c93d796b92f3 2024-11-15T23:17:06,816 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/4c9fdc07b4b544eb88e921bc250a8245 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/4c9fdc07b4b544eb88e921bc250a8245 2024-11-15T23:17:06,817 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/ea80aaad67994b6c9cae3084bb81e081 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/ea80aaad67994b6c9cae3084bb81e081 2024-11-15T23:17:06,818 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/95ce107556f445c3b50c1900ac3615c9 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/95ce107556f445c3b50c1900ac3615c9 2024-11-15T23:17:06,819 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/c2c9d85e3c6847159a8e76ea4598202a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/c2c9d85e3c6847159a8e76ea4598202a 2024-11-15T23:17:06,820 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/edf8622e6bee450e9c5ccad5e9763e84 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/edf8622e6bee450e9c5ccad5e9763e84 2024-11-15T23:17:06,820 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/e870cb012db64b7580b75b365dd7db7a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/e870cb012db64b7580b75b365dd7db7a 2024-11-15T23:17:06,821 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/8f1a7614580144ad874f81094fe1051e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/8f1a7614580144ad874f81094fe1051e 2024-11-15T23:17:06,927 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/0a0fb2700187406d95b709a51dddca74 2024-11-15T23:17:06,940 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/c0ba78d2738f43c38abce0155a705bdc is 50, key is test_row_1/B:col10/1731712625926/Put/seqid=0 2024-11-15T23:17:06,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742256_1432 (size=9857) 2024-11-15T23:17:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T23:17:07,344 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/c0ba78d2738f43c38abce0155a705bdc 2024-11-15T23:17:07,356 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/38a473a009c1448e8760915c1831c94a is 50, key is test_row_1/C:col10/1731712625926/Put/seqid=0 2024-11-15T23:17:07,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742257_1433 (size=9857) 2024-11-15T23:17:07,761 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/38a473a009c1448e8760915c1831c94a 2024-11-15T23:17:07,768 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/A/0a0fb2700187406d95b709a51dddca74 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/0a0fb2700187406d95b709a51dddca74 2024-11-15T23:17:07,773 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/0a0fb2700187406d95b709a51dddca74, entries=100, sequenceid=322, filesize=22.0 K 2024-11-15T23:17:07,774 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/B/c0ba78d2738f43c38abce0155a705bdc as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/c0ba78d2738f43c38abce0155a705bdc 2024-11-15T23:17:07,777 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/c0ba78d2738f43c38abce0155a705bdc, entries=100, sequenceid=322, filesize=9.6 K 2024-11-15T23:17:07,777 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/.tmp/C/38a473a009c1448e8760915c1831c94a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/38a473a009c1448e8760915c1831c94a 2024-11-15T23:17:07,780 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/38a473a009c1448e8760915c1831c94a, entries=100, sequenceid=322, filesize=9.6 K 2024-11-15T23:17:07,781 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2e9ed115eee55c880c6a2e34cdbb31c0 in 1689ms, sequenceid=322, compaction requested=true 2024-11-15T23:17:07,784 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/recovered.edits/325.seqid, newMaxSeqId=325, maxSeqId=4 2024-11-15T23:17:07,785 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. 2024-11-15T23:17:07,785 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for 2e9ed115eee55c880c6a2e34cdbb31c0: Waiting for close lock at 1731712626091Running coprocessor pre-close hooks at 1731712626091Disabling compacts and flushes for region at 1731712626091Disabling writes for close at 1731712626092 (+1 ms)Obtaining lock to block concurrent updates at 1731712626092Preparing flush snapshotting stores in 2e9ed115eee55c880c6a2e34cdbb31c0 at 1731712626092Finished memstore snapshotting TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731712626093 (+1 ms)Flushing stores of TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0. at 1731712626094 (+1 ms)Flushing 2e9ed115eee55c880c6a2e34cdbb31c0/A: creating writer at 1731712626094Flushing mob file 2e9ed115eee55c880c6a2e34cdbb31c0/A: appending metadata at 1731712626102 (+8 ms)Flushing mob file 2e9ed115eee55c880c6a2e34cdbb31c0/A: closing flushed file at 1731712626102Flushing 2e9ed115eee55c880c6a2e34cdbb31c0/A: appending metadata at 1731712626520 (+418 ms)Flushing 2e9ed115eee55c880c6a2e34cdbb31c0/A: closing flushed file at 1731712626520Flushing 2e9ed115eee55c880c6a2e34cdbb31c0/B: creating writer at 1731712626936 (+416 ms)Flushing 2e9ed115eee55c880c6a2e34cdbb31c0/B: appending metadata at 1731712626939 (+3 ms)Flushing 2e9ed115eee55c880c6a2e34cdbb31c0/B: closing flushed file at 1731712626939Flushing 2e9ed115eee55c880c6a2e34cdbb31c0/C: creating writer at 1731712627352 (+413 ms)Flushing 2e9ed115eee55c880c6a2e34cdbb31c0/C: appending metadata at 1731712627356 (+4 ms)Flushing 2e9ed115eee55c880c6a2e34cdbb31c0/C: closing flushed file at 1731712627356Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70b13aed: reopening flushed file at 1731712627767 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53011bb2: reopening flushed file at 1731712627773 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3458664: reopening flushed file at 1731712627777 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2e9ed115eee55c880c6a2e34cdbb31c0 in 1689ms, sequenceid=322, compaction requested=true at 1731712627781 (+4 ms)Writing region close event to WAL at 1731712627782 (+1 ms)Running coprocessor post-close hooks at 1731712627785 (+3 ms)Closed at 1731712627785 2024-11-15T23:17:07,787 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed 2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:07,787 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=2e9ed115eee55c880c6a2e34cdbb31c0, regionState=CLOSED 2024-11-15T23:17:07,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 because future has completed 2024-11-15T23:17:07,790 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-15T23:17:07,790 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure 2e9ed115eee55c880c6a2e34cdbb31c0, server=d6ecb9b7dda1,38353,1731712503763 in 1.8500 sec 2024-11-15T23:17:07,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-15T23:17:07,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9ed115eee55c880c6a2e34cdbb31c0, UNASSIGN in 1.8540 sec 2024-11-15T23:17:07,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-15T23:17:07,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8570 sec 2024-11-15T23:17:07,794 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712627794"}]},"ts":"1731712627794"} 2024-11-15T23:17:07,795 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T23:17:07,795 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T23:17:07,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8660 sec 2024-11-15T23:17:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T23:17:08,077 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:08,078 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T23:17:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:17:08,082 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:17:08,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T23:17:08,084 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:17:08,088 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:08,091 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A, FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B, FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C, FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/recovered.edits] 2024-11-15T23:17:08,095 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/0a0fb2700187406d95b709a51dddca74 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/0a0fb2700187406d95b709a51dddca74 2024-11-15T23:17:08,096 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/73c36801cbda40fb8ac47936e7048ab6 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/73c36801cbda40fb8ac47936e7048ab6 2024-11-15T23:17:08,097 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/9bfc666e2f824928acb27e2fdda145fb to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/9bfc666e2f824928acb27e2fdda145fb 2024-11-15T23:17:08,098 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/b56c7c0be4ce4efdb9735f11c42d47a5 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/A/b56c7c0be4ce4efdb9735f11c42d47a5 2024-11-15T23:17:08,100 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/10b4c9d0caf74a09b24399836528bf29 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/10b4c9d0caf74a09b24399836528bf29 2024-11-15T23:17:08,101 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/212ddb4b344c41dfafc9788fe70063da to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/212ddb4b344c41dfafc9788fe70063da 2024-11-15T23:17:08,102 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/c0ba78d2738f43c38abce0155a705bdc to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/c0ba78d2738f43c38abce0155a705bdc 2024-11-15T23:17:08,103 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/d52b289760034d01bea381afc9d6b836 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/B/d52b289760034d01bea381afc9d6b836 2024-11-15T23:17:08,104 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/0189edca291f42f98472999ffcdc7ae8 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/0189edca291f42f98472999ffcdc7ae8 2024-11-15T23:17:08,105 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/38a473a009c1448e8760915c1831c94a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/38a473a009c1448e8760915c1831c94a 2024-11-15T23:17:08,106 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/c491ebe5cfcd416da3fd9c0df891a9bc to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/c491ebe5cfcd416da3fd9c0df891a9bc 2024-11-15T23:17:08,108 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/d46df57e5514420e995122eaff36184f to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/C/d46df57e5514420e995122eaff36184f 2024-11-15T23:17:08,110 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/recovered.edits/325.seqid to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0/recovered.edits/325.seqid 2024-11-15T23:17:08,111 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:08,111 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T23:17:08,112 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T23:17:08,113 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T23:17:08,116 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150065f5527bb64fde8a22f7d65bd0e212_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150065f5527bb64fde8a22f7d65bd0e212_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:08,117 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111537018b3606bf4b978c77768c378d93ca_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111537018b3606bf4b978c77768c378d93ca_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:08,118 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111538e64df05f21492897771c0d1df58bad_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111538e64df05f21492897771c0d1df58bad_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:08,119 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111553373645acab489cb4b70c72e3c71e26_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111553373645acab489cb4b70c72e3c71e26_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:08,120 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157b99c6b732f6423fbf106f928820477a_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157b99c6b732f6423fbf106f928820477a_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:08,121 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115815fbfaaa1ee408fab1c4a3a70bc1fa5_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115815fbfaaa1ee408fab1c4a3a70bc1fa5_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:08,122 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158d90e863969141f9b566c68d384ddeb9_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158d90e863969141f9b566c68d384ddeb9_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:08,123 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a2790e94994e472a844d1db477efe32f_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a2790e94994e472a844d1db477efe32f_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:08,124 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b44f43bac6df4653afb14e694c0355a0_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b44f43bac6df4653afb14e694c0355a0_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:08,125 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b798e211f1074636aa9f82238b943b21_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b798e211f1074636aa9f82238b943b21_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:08,126 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bd058e5963e5495a8933617e7b24a103_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bd058e5963e5495a8933617e7b24a103_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:08,127 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c17e9ffd63f54db0bf392b40b2f00382_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c17e9ffd63f54db0bf392b40b2f00382_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:08,128 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c8b624772e0e44e0b49a9e960ce7afbd_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c8b624772e0e44e0b49a9e960ce7afbd_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:08,129 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ddf747fac8124f529f03ded11865634a_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ddf747fac8124f529f03ded11865634a_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:08,130 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e3f5b789967046ce81196551d7a9737c_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e3f5b789967046ce81196551d7a9737c_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:08,131 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e82bc6f79d4a4769b419dda40e5787e6_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e82bc6f79d4a4769b419dda40e5787e6_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:08,132 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f018b2a8991c4d999eb609213caec43c_2e9ed115eee55c880c6a2e34cdbb31c0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f018b2a8991c4d999eb609213caec43c_2e9ed115eee55c880c6a2e34cdbb31c0 2024-11-15T23:17:08,133 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T23:17:08,134 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:17:08,136 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T23:17:08,138 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T23:17:08,139 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:17:08,139 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T23:17:08,139 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731712628139"}]},"ts":"9223372036854775807"} 2024-11-15T23:17:08,141 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T23:17:08,141 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2e9ed115eee55c880c6a2e34cdbb31c0, NAME => 'TestAcidGuarantees,,1731712598309.2e9ed115eee55c880c6a2e34cdbb31c0.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T23:17:08,141 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T23:17:08,141 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731712628141"}]},"ts":"9223372036854775807"} 2024-11-15T23:17:08,143 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T23:17:08,143 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:17:08,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 65 msec 2024-11-15T23:17:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T23:17:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T23:17:08,196 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:08,203 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249 (was 248) - Thread LEAK? -, OpenFileDescriptor=453 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=421 (was 468), ProcessCount=11 (was 11), AvailableMemoryMB=5525 (was 5559) 2024-11-15T23:17:08,210 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=421, ProcessCount=11, AvailableMemoryMB=5525 2024-11-15T23:17:08,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T23:17:08,211 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T23:17:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T23:17:08,213 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T23:17:08,213 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:08,213 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-11-15T23:17:08,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-15T23:17:08,214 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T23:17:08,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742258_1434 (size=1061) 2024-11-15T23:17:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-15T23:17:08,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-15T23:17:08,622 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444 2024-11-15T23:17:08,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742259_1435 (size=53) 2024-11-15T23:17:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-15T23:17:09,031 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:17:09,031 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3ad510e3715c6faa652057022b8e1f17, disabling compactions & flushes 2024-11-15T23:17:09,031 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:09,031 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:09,031 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. after waiting 0 ms 2024-11-15T23:17:09,031 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:09,031 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:09,031 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3ad510e3715c6faa652057022b8e1f17: Waiting for close lock at 1731712629031Disabling compacts and flushes for region at 1731712629031Disabling writes for close at 1731712629031Writing region close event to WAL at 1731712629031Closed at 1731712629031 2024-11-15T23:17:09,032 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T23:17:09,032 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731712629032"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731712629032"}]},"ts":"1731712629032"} 2024-11-15T23:17:09,034 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T23:17:09,034 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T23:17:09,035 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712629035"}]},"ts":"1731712629035"} 2024-11-15T23:17:09,036 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T23:17:09,036 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ad510e3715c6faa652057022b8e1f17, ASSIGN}] 2024-11-15T23:17:09,037 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ad510e3715c6faa652057022b8e1f17, ASSIGN 2024-11-15T23:17:09,038 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ad510e3715c6faa652057022b8e1f17, ASSIGN; state=OFFLINE, location=d6ecb9b7dda1,38353,1731712503763; forceNewPlan=false, retain=false 2024-11-15T23:17:09,189 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=3ad510e3715c6faa652057022b8e1f17, regionState=OPENING, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:09,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ad510e3715c6faa652057022b8e1f17, ASSIGN because future has completed 2024-11-15T23:17:09,193 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763}] 2024-11-15T23:17:09,356 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-15T23:17:09,356 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} 2024-11-15T23:17:09,357 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:09,357 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:17:09,358 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:09,358 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:09,360 INFO [StoreOpener-3ad510e3715c6faa652057022b8e1f17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:09,361 INFO [StoreOpener-3ad510e3715c6faa652057022b8e1f17-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:17:09,361 INFO [StoreOpener-3ad510e3715c6faa652057022b8e1f17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ad510e3715c6faa652057022b8e1f17 columnFamilyName A 2024-11-15T23:17:09,361 DEBUG [StoreOpener-3ad510e3715c6faa652057022b8e1f17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:09,361 INFO [StoreOpener-3ad510e3715c6faa652057022b8e1f17-1 {}] regionserver.HStore(327): Store=3ad510e3715c6faa652057022b8e1f17/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:17:09,361 INFO [StoreOpener-3ad510e3715c6faa652057022b8e1f17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:09,362 INFO [StoreOpener-3ad510e3715c6faa652057022b8e1f17-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:17:09,362 INFO [StoreOpener-3ad510e3715c6faa652057022b8e1f17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ad510e3715c6faa652057022b8e1f17 columnFamilyName B 2024-11-15T23:17:09,362 DEBUG [StoreOpener-3ad510e3715c6faa652057022b8e1f17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:09,363 INFO [StoreOpener-3ad510e3715c6faa652057022b8e1f17-1 {}] regionserver.HStore(327): Store=3ad510e3715c6faa652057022b8e1f17/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:17:09,363 INFO [StoreOpener-3ad510e3715c6faa652057022b8e1f17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:09,363 INFO [StoreOpener-3ad510e3715c6faa652057022b8e1f17-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:17:09,363 INFO [StoreOpener-3ad510e3715c6faa652057022b8e1f17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ad510e3715c6faa652057022b8e1f17 columnFamilyName C 2024-11-15T23:17:09,363 DEBUG [StoreOpener-3ad510e3715c6faa652057022b8e1f17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:09,364 INFO [StoreOpener-3ad510e3715c6faa652057022b8e1f17-1 {}] regionserver.HStore(327): Store=3ad510e3715c6faa652057022b8e1f17/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:17:09,364 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:09,364 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:09,364 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:09,365 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:09,365 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:09,365 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:09,366 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T23:17:09,366 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:09,368 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T23:17:09,368 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened 3ad510e3715c6faa652057022b8e1f17; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59567712, jitterRate=-0.11237192153930664}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T23:17:09,368 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:09,368 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for 3ad510e3715c6faa652057022b8e1f17: Running coprocessor pre-open hook at 1731712629358Writing region info on filesystem at 1731712629358Initializing all the Stores at 1731712629359 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712629359Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712629360 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712629360Cleaning up temporary data from old regions at 1731712629365 (+5 ms)Running coprocessor post-open hooks at 1731712629368 (+3 ms)Region opened successfully at 1731712629368 2024-11-15T23:17:09,369 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., pid=114, masterSystemTime=1731712629348 2024-11-15T23:17:09,370 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:09,370 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:09,371 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=3ad510e3715c6faa652057022b8e1f17, regionState=OPEN, openSeqNum=2, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:09,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 because future has completed 2024-11-15T23:17:09,374 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-15T23:17:09,374 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure 3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 in 180 msec 2024-11-15T23:17:09,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-15T23:17:09,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ad510e3715c6faa652057022b8e1f17, ASSIGN in 338 msec 2024-11-15T23:17:09,376 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T23:17:09,376 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712629376"}]},"ts":"1731712629376"} 2024-11-15T23:17:09,378 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T23:17:09,378 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T23:17:09,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-11-15T23:17:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-15T23:17:10,366 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:10,368 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d55bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,368 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:10,368 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:10,369 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:10,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:10,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:10,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d801af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:10,371 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:10,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@613962f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,372 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:10,374 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:10,375 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T23:17:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is d6ecb9b7dda1,43421,1731712502855 2024-11-15T23:17:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@322986ee 2024-11-15T23:17:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T23:17:10,377 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T23:17:10,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,379 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:10,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:10,380 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:10,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:10,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b89ab3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:10,382 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:10,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@84b3f07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,384 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:10,384 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:10,385 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f2b1af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:10,386 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:10,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef94985, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,388 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,388 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:10,388 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:10,389 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:10,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:10,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@584225dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:10,391 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@317226da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,393 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:10,393 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:10,394 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@106a624d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:10,395 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:10,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e7fd25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,397 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:10,397 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:10,398 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:10,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@752ea842, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:10,400 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:10,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b4c51b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,401 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:10,402 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:10,402 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:10,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f47ee6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:10,404 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5db4d4db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,406 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:10,406 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:10,407 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32d6e67c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:10,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:10,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:10,409 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:10,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31f8c165, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,410 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:10,410 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:10,411 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:10,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@82b3ab7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:10,413 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:10,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f10153, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,414 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:10,414 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:10,414 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf2498f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:10,416 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25f1c210, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,417 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:10,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:10,417 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:10,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:10,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:10,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58c44425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:10,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:10,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:10,419 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:10,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e8a4b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:10,420 DEBUG [Thread-1960 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:10,420 DEBUG [Thread-1975 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:10,420 DEBUG [Thread-1964 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:10,421 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:17:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:10,421 DEBUG [Thread-1962 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:10,421 DEBUG [Thread-1971 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-15T23:17:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:10,422 DEBUG [Thread-1977 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:10,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:10,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:10,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T23:17:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:10,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:17:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:17:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:10,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:10,425 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:17:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:17:10,425 DEBUG [Thread-1979 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:17:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:10,427 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:10,427 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:17:10,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:17:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:17:10,427 DEBUG [Thread-1966 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:10,428 DEBUG [Thread-1973 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:10,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:17:10,428 DEBUG [Thread-1968 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:10,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:10,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:10,429 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:10,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:17:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:17:10,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:10,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:17:10,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2] 2024-11-15T23:17:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:10,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T23:17:10,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:10,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:10,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:10,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:10,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:10,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:10,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/113b8928f8fe47438033e9cdc8e2f0f3 is 50, key is test_row_0/A:col10/1731712630426/Put/seqid=0 2024-11-15T23:17:10,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742260_1436 (size=16681) 2024-11-15T23:17:10,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712690469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712690474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:10,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712690476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:10,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:10,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712690477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:10,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:10,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:10,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712690477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:10,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T23:17:10,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T23:17:10,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:10,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:10,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:10,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:10,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:10,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712690585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:10,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712690585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:10,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712690585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:10,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712690595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712690595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T23:17:10,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T23:17:10,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:10,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:10,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:10,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:10,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:10,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:10,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712690795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712690796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:10,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712690796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:10,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712690805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712690805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:10,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:10,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:10,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/113b8928f8fe47438033e9cdc8e2f0f3 2024-11-15T23:17:10,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T23:17:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:10,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:10,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/4e29bbd020f24bc0975d170dc4749d04 is 50, key is test_row_0/B:col10/1731712630426/Put/seqid=0 2024-11-15T23:17:10,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742261_1437 (size=12001) 2024-11-15T23:17:10,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/4e29bbd020f24bc0975d170dc4749d04 2024-11-15T23:17:10,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/9d2227b1c3544551af1633c8ec33f640 is 50, key is test_row_0/C:col10/1731712630426/Put/seqid=0 2024-11-15T23:17:10,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742262_1438 (size=12001) 2024-11-15T23:17:11,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T23:17:11,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:11,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:11,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:11,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:11,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T23:17:11,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712691105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712691105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:11,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712691105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:11,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712691115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:11,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712691126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:11,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:11,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T23:17:11,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:11,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:11,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:11,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:11,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:11,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/9d2227b1c3544551af1633c8ec33f640 2024-11-15T23:17:11,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/113b8928f8fe47438033e9cdc8e2f0f3 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/113b8928f8fe47438033e9cdc8e2f0f3 2024-11-15T23:17:11,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/113b8928f8fe47438033e9cdc8e2f0f3, entries=250, sequenceid=13, filesize=16.3 K 2024-11-15T23:17:11,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/4e29bbd020f24bc0975d170dc4749d04 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/4e29bbd020f24bc0975d170dc4749d04 2024-11-15T23:17:11,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/4e29bbd020f24bc0975d170dc4749d04, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T23:17:11,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/9d2227b1c3544551af1633c8ec33f640 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/9d2227b1c3544551af1633c8ec33f640 2024-11-15T23:17:11,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T23:17:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:11,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:11,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/9d2227b1c3544551af1633c8ec33f640, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T23:17:11,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 3ad510e3715c6faa652057022b8e1f17 in 917ms, sequenceid=13, compaction requested=false 2024-11-15T23:17:11,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:11,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T23:17:11,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:11,503 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T23:17:11,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:11,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:11,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:11,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:11,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:11,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:11,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/c6bd8df6929549e7a0547d2abb4f4a9b is 50, key is test_row_0/A:col10/1731712630474/Put/seqid=0 2024-11-15T23:17:11,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742263_1439 (size=12001) 2024-11-15T23:17:11,511 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/c6bd8df6929549e7a0547d2abb4f4a9b 2024-11-15T23:17:11,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/ad83e68be27749968c65827d76f7ccd9 is 50, key is test_row_0/B:col10/1731712630474/Put/seqid=0 2024-11-15T23:17:11,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742264_1440 (size=12001) 2024-11-15T23:17:11,536 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/ad83e68be27749968c65827d76f7ccd9 2024-11-15T23:17:11,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/e57ca7ca2fd842278e8f83f8e913a523 is 50, key is test_row_0/C:col10/1731712630474/Put/seqid=0 2024-11-15T23:17:11,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742265_1441 (size=12001) 2024-11-15T23:17:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T23:17:11,560 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/e57ca7ca2fd842278e8f83f8e913a523 2024-11-15T23:17:11,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/c6bd8df6929549e7a0547d2abb4f4a9b as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/c6bd8df6929549e7a0547d2abb4f4a9b 2024-11-15T23:17:11,567 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/c6bd8df6929549e7a0547d2abb4f4a9b, entries=150, sequenceid=33, filesize=11.7 K 2024-11-15T23:17:11,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/ad83e68be27749968c65827d76f7ccd9 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/ad83e68be27749968c65827d76f7ccd9 2024-11-15T23:17:11,571 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/ad83e68be27749968c65827d76f7ccd9, entries=150, sequenceid=33, filesize=11.7 K 2024-11-15T23:17:11,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/e57ca7ca2fd842278e8f83f8e913a523 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/e57ca7ca2fd842278e8f83f8e913a523 2024-11-15T23:17:11,575 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/e57ca7ca2fd842278e8f83f8e913a523, entries=150, sequenceid=33, filesize=11.7 K 2024-11-15T23:17:11,576 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 3ad510e3715c6faa652057022b8e1f17 in 73ms, sequenceid=33, compaction requested=false 2024-11-15T23:17:11,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:11,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:11,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-15T23:17:11,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-15T23:17:11,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-15T23:17:11,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1500 sec 2024-11-15T23:17:11,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.1590 sec 2024-11-15T23:17:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:11,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T23:17:11,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:11,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:11,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:11,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:11,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:11,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:11,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/2b645254f0c8499a8dd705aadb491535 is 50, key is test_row_0/A:col10/1731712631632/Put/seqid=0 2024-11-15T23:17:11,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742266_1442 (size=12001) 2024-11-15T23:17:11,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731712631661, max timestamp=1731712631686Num uniques 859; 2024-11-15T23:17:11,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T23:17:11,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731712631661, max timestamp=1731712631686Num uniques 859; 2024-11-15T23:17:11,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731712631661, max timestamp=1731712631686Num uniques 859; 2024-11-15T23:17:11,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T23:17:11,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T23:17:11,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712691720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:11,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:11,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712691730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:11,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712691731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:11,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712691732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:11,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:11,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712691735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:11,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:11,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712691845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:11,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:11,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712691845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:11,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712691845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:11,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712691855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:11,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:11,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712691855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:11,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:11,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:12,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/2b645254f0c8499a8dd705aadb491535 2024-11-15T23:17:12,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/cad197f69ab344feb578c51ccf03e0c8 is 50, key is test_row_0/B:col10/1731712631632/Put/seqid=0 2024-11-15T23:17:12,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742267_1443 (size=12001) 2024-11-15T23:17:12,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712692065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:12,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:12,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712692065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:12,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712692065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:12,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712692076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712692076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:12,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712692385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:12,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712692385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712692385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:12,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712692395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712692395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:12,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/cad197f69ab344feb578c51ccf03e0c8 2024-11-15T23:17:12,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/ccc087c662bb47bf887ec3e97e8bb21a is 50, key is test_row_0/C:col10/1731712631632/Put/seqid=0 2024-11-15T23:17:12,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742268_1444 (size=12001) 2024-11-15T23:17:12,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/ccc087c662bb47bf887ec3e97e8bb21a 2024-11-15T23:17:12,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/2b645254f0c8499a8dd705aadb491535 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/2b645254f0c8499a8dd705aadb491535 2024-11-15T23:17:12,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/2b645254f0c8499a8dd705aadb491535, entries=150, sequenceid=44, filesize=11.7 K 2024-11-15T23:17:12,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/cad197f69ab344feb578c51ccf03e0c8 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/cad197f69ab344feb578c51ccf03e0c8 2024-11-15T23:17:12,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/cad197f69ab344feb578c51ccf03e0c8, entries=150, sequenceid=44, filesize=11.7 K 2024-11-15T23:17:12,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/ccc087c662bb47bf887ec3e97e8bb21a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/ccc087c662bb47bf887ec3e97e8bb21a 2024-11-15T23:17:12,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/ccc087c662bb47bf887ec3e97e8bb21a, entries=150, sequenceid=44, filesize=11.7 K 2024-11-15T23:17:12,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 3ad510e3715c6faa652057022b8e1f17 in 867ms, sequenceid=44, compaction requested=true 2024-11-15T23:17:12,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:12,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:17:12,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:12,500 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:12,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:17:12,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:12,500 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:12,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:17:12,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:12,500 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:12,500 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:12,500 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/B is initiating minor compaction (all files) 2024-11-15T23:17:12,500 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/A is initiating minor compaction (all files) 2024-11-15T23:17:12,500 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/A in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:12,500 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/B in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:12,500 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/113b8928f8fe47438033e9cdc8e2f0f3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/c6bd8df6929549e7a0547d2abb4f4a9b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/2b645254f0c8499a8dd705aadb491535] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=39.7 K 2024-11-15T23:17:12,500 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/4e29bbd020f24bc0975d170dc4749d04, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/ad83e68be27749968c65827d76f7ccd9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/cad197f69ab344feb578c51ccf03e0c8] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=35.2 K 2024-11-15T23:17:12,501 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 113b8928f8fe47438033e9cdc8e2f0f3, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731712630426 2024-11-15T23:17:12,501 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e29bbd020f24bc0975d170dc4749d04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731712630426 2024-11-15T23:17:12,501 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6bd8df6929549e7a0547d2abb4f4a9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731712630461 2024-11-15T23:17:12,501 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting ad83e68be27749968c65827d76f7ccd9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731712630461 2024-11-15T23:17:12,501 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b645254f0c8499a8dd705aadb491535, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1731712631619 2024-11-15T23:17:12,501 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting cad197f69ab344feb578c51ccf03e0c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1731712631619 2024-11-15T23:17:12,507 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#B#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:12,507 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/2a513c0256e5410684b637b414f4b27d is 50, key is test_row_0/B:col10/1731712631632/Put/seqid=0 2024-11-15T23:17:12,508 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#A#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:12,508 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/ba7b94c3bd5d4afeb7bdaddbd63e2a64 is 50, key is test_row_0/A:col10/1731712631632/Put/seqid=0 2024-11-15T23:17:12,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742269_1445 (size=12104) 2024-11-15T23:17:12,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742270_1446 (size=12104) 2024-11-15T23:17:12,520 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/2a513c0256e5410684b637b414f4b27d as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/2a513c0256e5410684b637b414f4b27d 2024-11-15T23:17:12,523 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/B of 3ad510e3715c6faa652057022b8e1f17 into 2a513c0256e5410684b637b414f4b27d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:12,523 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:12,523 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/B, priority=13, startTime=1731712632500; duration=0sec 2024-11-15T23:17:12,523 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:12,523 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:B 2024-11-15T23:17:12,523 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:12,524 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:12,524 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/C is initiating minor compaction (all files) 2024-11-15T23:17:12,524 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/C in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:12,524 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/9d2227b1c3544551af1633c8ec33f640, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/e57ca7ca2fd842278e8f83f8e913a523, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/ccc087c662bb47bf887ec3e97e8bb21a] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=35.2 K 2024-11-15T23:17:12,524 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d2227b1c3544551af1633c8ec33f640, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731712630426 2024-11-15T23:17:12,524 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting e57ca7ca2fd842278e8f83f8e913a523, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731712630461 2024-11-15T23:17:12,525 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting ccc087c662bb47bf887ec3e97e8bb21a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1731712631619 2024-11-15T23:17:12,530 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#C#compaction#373 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:12,531 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/789ac68bdef54405b548db65e58dec5e is 50, key is test_row_0/C:col10/1731712631632/Put/seqid=0 2024-11-15T23:17:12,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742271_1447 (size=12104) 2024-11-15T23:17:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T23:17:12,565 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:12,568 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:17:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-15T23:17:12,570 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:17:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T23:17:12,570 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:17:12,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:17:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T23:17:12,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T23:17:12,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:12,723 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-15T23:17:12,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:12,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:12,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:12,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:12,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:12,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:12,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/7835944832a8495bb8a4de2f380a2237 is 50, key is test_row_0/A:col10/1731712631728/Put/seqid=0 2024-11-15T23:17:12,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T23:17:12,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742272_1448 (size=12001) 2024-11-15T23:17:12,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:12,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:12,921 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/ba7b94c3bd5d4afeb7bdaddbd63e2a64 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/ba7b94c3bd5d4afeb7bdaddbd63e2a64 2024-11-15T23:17:12,924 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/A of 3ad510e3715c6faa652057022b8e1f17 into ba7b94c3bd5d4afeb7bdaddbd63e2a64(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:12,924 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:12,924 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/A, priority=13, startTime=1731712632499; duration=0sec 2024-11-15T23:17:12,924 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:12,924 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:A 2024-11-15T23:17:12,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712692914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:12,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712692915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:12,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:12,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712692919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:12,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:12,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:12,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712692928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:12,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:12,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712692929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:12,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:12,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:12,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:12,937 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/789ac68bdef54405b548db65e58dec5e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/789ac68bdef54405b548db65e58dec5e 2024-11-15T23:17:12,941 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/C of 3ad510e3715c6faa652057022b8e1f17 into 789ac68bdef54405b548db65e58dec5e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:12,941 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:12,941 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/C, priority=13, startTime=1731712632500; duration=0sec 2024-11-15T23:17:12,941 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:12,941 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:C 2024-11-15T23:17:13,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:13,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712693035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:13,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:13,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712693035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:13,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712693035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:13,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712693035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:13,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712693035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T23:17:13,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:13,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712693255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:13,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:13,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712693255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:13,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:13,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712693255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:13,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:13,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712693255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:13,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:13,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:13,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712693256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:13,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:13,290 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/7835944832a8495bb8a4de2f380a2237 2024-11-15T23:17:13,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/c3630c6cf49f4c92899207d1f897eff2 is 50, key is test_row_0/B:col10/1731712631728/Put/seqid=0 2024-11-15T23:17:13,301 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T23:17:13,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742273_1449 (size=12001) 2024-11-15T23:17:13,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712693565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:13,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:13,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712693565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:13,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:13,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:13,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712693565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:13,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:13,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:13,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712693575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712693575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:13,703 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/c3630c6cf49f4c92899207d1f897eff2 2024-11-15T23:17:13,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T23:17:13,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/4ef4cc7a28bd4c0390db2e1458462d54 is 50, key is test_row_0/C:col10/1731712631728/Put/seqid=0 2024-11-15T23:17:13,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742274_1450 (size=12001) 2024-11-15T23:17:14,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:14,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712694075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712694075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:14,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712694087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:14,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712694095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:14,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:14,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:14,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712694095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:14,112 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/4ef4cc7a28bd4c0390db2e1458462d54 2024-11-15T23:17:14,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/7835944832a8495bb8a4de2f380a2237 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/7835944832a8495bb8a4de2f380a2237 2024-11-15T23:17:14,117 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/7835944832a8495bb8a4de2f380a2237, entries=150, sequenceid=79, filesize=11.7 K 2024-11-15T23:17:14,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-15T23:17:14,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/c3630c6cf49f4c92899207d1f897eff2 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/c3630c6cf49f4c92899207d1f897eff2 2024-11-15T23:17:14,121 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/c3630c6cf49f4c92899207d1f897eff2, entries=150, sequenceid=79, filesize=11.7 K 2024-11-15T23:17:14,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/4ef4cc7a28bd4c0390db2e1458462d54 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4ef4cc7a28bd4c0390db2e1458462d54 2024-11-15T23:17:14,123 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4ef4cc7a28bd4c0390db2e1458462d54, entries=150, sequenceid=79, filesize=11.7 K 2024-11-15T23:17:14,124 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 3ad510e3715c6faa652057022b8e1f17 in 1401ms, sequenceid=79, compaction requested=false 2024-11-15T23:17:14,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:14,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:14,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-15T23:17:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-15T23:17:14,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-15T23:17:14,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5550 sec 2024-11-15T23:17:14,129 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.5600 sec 2024-11-15T23:17:14,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T23:17:14,716 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:14,716 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:17:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-15T23:17:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T23:17:14,718 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:17:14,718 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:17:14,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:17:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T23:17:14,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T23:17:14,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:14,871 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T23:17:14,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:14,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:14,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:14,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:14,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:14,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:14,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/0361f1f97afd4252bdc54ca44d6c4a0c is 50, key is test_row_0/A:col10/1731712632927/Put/seqid=0 2024-11-15T23:17:14,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742275_1451 (size=12001) 2024-11-15T23:17:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T23:17:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:15,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:15,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712695124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712695126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731712635109, max timestamp=1731712635134Num uniques 859; 2024-11-15T23:17:15,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731712635109, max timestamp=1731712635134Num uniques 859; 2024-11-15T23:17:15,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712695134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T23:17:15,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T23:17:15,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731712635109, max timestamp=1731712635134Num uniques 859; 2024-11-15T23:17:15,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T23:17:15,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712695184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712695191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712695245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712695245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712695246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,279 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/0361f1f97afd4252bdc54ca44d6c4a0c 2024-11-15T23:17:15,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/9139baef26064fda80175fb26658f333 is 50, key is test_row_0/B:col10/1731712632927/Put/seqid=0 2024-11-15T23:17:15,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742276_1452 (size=12001) 2024-11-15T23:17:15,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712695305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712695305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T23:17:15,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712695466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712695466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712695466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712695515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712695515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,687 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/9139baef26064fda80175fb26658f333 2024-11-15T23:17:15,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/33b0d42b6ce849ecbd43f7d71ff8757c is 50, key is test_row_0/C:col10/1731712632927/Put/seqid=0 2024-11-15T23:17:15,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742277_1453 (size=12001) 2024-11-15T23:17:15,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712695785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712695786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712695786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712695835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712695835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T23:17:16,100 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/33b0d42b6ce849ecbd43f7d71ff8757c 2024-11-15T23:17:16,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/0361f1f97afd4252bdc54ca44d6c4a0c as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/0361f1f97afd4252bdc54ca44d6c4a0c 2024-11-15T23:17:16,106 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/0361f1f97afd4252bdc54ca44d6c4a0c, entries=150, sequenceid=92, filesize=11.7 K 2024-11-15T23:17:16,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/9139baef26064fda80175fb26658f333 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/9139baef26064fda80175fb26658f333 2024-11-15T23:17:16,110 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/9139baef26064fda80175fb26658f333, entries=150, sequenceid=92, filesize=11.7 K 2024-11-15T23:17:16,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/33b0d42b6ce849ecbd43f7d71ff8757c as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/33b0d42b6ce849ecbd43f7d71ff8757c 2024-11-15T23:17:16,114 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/33b0d42b6ce849ecbd43f7d71ff8757c, entries=150, sequenceid=92, filesize=11.7 K 2024-11-15T23:17:16,114 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 3ad510e3715c6faa652057022b8e1f17 in 1243ms, sequenceid=92, compaction requested=true 2024-11-15T23:17:16,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:16,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:16,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-15T23:17:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-15T23:17:16,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-15T23:17:16,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3980 sec 2024-11-15T23:17:16,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.4010 sec 2024-11-15T23:17:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:16,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-15T23:17:16,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:16,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:16,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:16,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:16,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:16,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:16,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/06ca1cfb449a4a6a875658e0ed034b09 is 50, key is test_row_0/A:col10/1731712635180/Put/seqid=0 2024-11-15T23:17:16,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742278_1454 (size=14341) 2024-11-15T23:17:16,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/06ca1cfb449a4a6a875658e0ed034b09 2024-11-15T23:17:16,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/640055d7dc27472b9f08becfb2b3b2bc is 50, key is test_row_0/B:col10/1731712635180/Put/seqid=0 2024-11-15T23:17:16,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742279_1455 (size=12001) 2024-11-15T23:17:16,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712696325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:16,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712696329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:16,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:16,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:16,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712696341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:16,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:16,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712696346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:16,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712696346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:16,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:16,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:16,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712696445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:16,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712696446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:16,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:16,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712696455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:16,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:16,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712696655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:16,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:16,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712696655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:16,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:16,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:16,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712696665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:16,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:16,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/640055d7dc27472b9f08becfb2b3b2bc 2024-11-15T23:17:16,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/195f597632164ca5a6c8a0202eedc709 is 50, key is test_row_0/C:col10/1731712635180/Put/seqid=0 2024-11-15T23:17:16,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742280_1456 (size=12001) 2024-11-15T23:17:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T23:17:16,866 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:16,866 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:17:16,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-15T23:17:16,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T23:17:16,868 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:17:16,868 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:17:16,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:17:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T23:17:16,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712696968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:16,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:16,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712696968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:16,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:16,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:16,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712696975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:17,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T23:17:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:17,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:17,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/195f597632164ca5a6c8a0202eedc709 2024-11-15T23:17:17,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/06ca1cfb449a4a6a875658e0ed034b09 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/06ca1cfb449a4a6a875658e0ed034b09 2024-11-15T23:17:17,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/06ca1cfb449a4a6a875658e0ed034b09, entries=200, sequenceid=127, filesize=14.0 K 2024-11-15T23:17:17,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/640055d7dc27472b9f08becfb2b3b2bc as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/640055d7dc27472b9f08becfb2b3b2bc 2024-11-15T23:17:17,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/640055d7dc27472b9f08becfb2b3b2bc, entries=150, sequenceid=127, filesize=11.7 K 2024-11-15T23:17:17,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/195f597632164ca5a6c8a0202eedc709 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/195f597632164ca5a6c8a0202eedc709 2024-11-15T23:17:17,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/195f597632164ca5a6c8a0202eedc709, entries=150, sequenceid=127, filesize=11.7 K 2024-11-15T23:17:17,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 3ad510e3715c6faa652057022b8e1f17 in 856ms, sequenceid=127, compaction requested=true 2024-11-15T23:17:17,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:17,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:17:17,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:17,161 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:17:17,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:17:17,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:17,161 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:17:17,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:17:17,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:17,162 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:17:17,162 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:17:17,162 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/B is initiating minor compaction (all files) 2024-11-15T23:17:17,162 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/A is initiating minor compaction (all files) 2024-11-15T23:17:17,162 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/B in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:17,162 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/A in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:17,162 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/ba7b94c3bd5d4afeb7bdaddbd63e2a64, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/7835944832a8495bb8a4de2f380a2237, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/0361f1f97afd4252bdc54ca44d6c4a0c, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/06ca1cfb449a4a6a875658e0ed034b09] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=49.3 K 2024-11-15T23:17:17,162 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/2a513c0256e5410684b637b414f4b27d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/c3630c6cf49f4c92899207d1f897eff2, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/9139baef26064fda80175fb26658f333, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/640055d7dc27472b9f08becfb2b3b2bc] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=47.0 K 2024-11-15T23:17:17,162 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a513c0256e5410684b637b414f4b27d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1731712631619 2024-11-15T23:17:17,162 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba7b94c3bd5d4afeb7bdaddbd63e2a64, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1731712631619 2024-11-15T23:17:17,166 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting c3630c6cf49f4c92899207d1f897eff2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731712631728 2024-11-15T23:17:17,166 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7835944832a8495bb8a4de2f380a2237, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731712631728 2024-11-15T23:17:17,166 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0361f1f97afd4252bdc54ca44d6c4a0c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731712632911 2024-11-15T23:17:17,167 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06ca1cfb449a4a6a875658e0ed034b09, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731712635180 2024-11-15T23:17:17,167 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 9139baef26064fda80175fb26658f333, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731712632911 2024-11-15T23:17:17,169 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 640055d7dc27472b9f08becfb2b3b2bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731712635180 2024-11-15T23:17:17,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T23:17:17,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:17,173 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T23:17:17,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:17,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:17,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:17,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:17,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:17,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T23:17:17,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/bdabe2ae55c54b8fa027864ae940470c is 50, key is test_row_0/A:col10/1731712636322/Put/seqid=0 2024-11-15T23:17:17,211 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#A#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:17,211 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#B#compaction#385 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:17,211 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/f5aee2c6cab148f6afe6cdc782e62bd1 is 50, key is test_row_0/A:col10/1731712635180/Put/seqid=0 2024-11-15T23:17:17,211 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/9a88b40ccd0b4694979f9d60ffe47c81 is 50, key is test_row_0/B:col10/1731712635180/Put/seqid=0 2024-11-15T23:17:17,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742281_1457 (size=12151) 2024-11-15T23:17:17,224 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/bdabe2ae55c54b8fa027864ae940470c 2024-11-15T23:17:17,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/3a6d4d071a614b05ae4305c7f724a0c6 is 50, key is test_row_0/B:col10/1731712636322/Put/seqid=0 2024-11-15T23:17:17,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742282_1458 (size=12241) 2024-11-15T23:17:17,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742283_1459 (size=12241) 2024-11-15T23:17:17,311 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/f5aee2c6cab148f6afe6cdc782e62bd1 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/f5aee2c6cab148f6afe6cdc782e62bd1 2024-11-15T23:17:17,320 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/A of 3ad510e3715c6faa652057022b8e1f17 into f5aee2c6cab148f6afe6cdc782e62bd1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:17,320 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:17,320 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/A, priority=12, startTime=1731712637161; duration=0sec 2024-11-15T23:17:17,320 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:17,320 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:A 2024-11-15T23:17:17,320 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:17:17,321 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:17:17,321 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/C is initiating minor compaction (all files) 2024-11-15T23:17:17,321 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/C in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:17,321 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/789ac68bdef54405b548db65e58dec5e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4ef4cc7a28bd4c0390db2e1458462d54, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/33b0d42b6ce849ecbd43f7d71ff8757c, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/195f597632164ca5a6c8a0202eedc709] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=47.0 K 2024-11-15T23:17:17,322 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 789ac68bdef54405b548db65e58dec5e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1731712631619 2024-11-15T23:17:17,322 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ef4cc7a28bd4c0390db2e1458462d54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731712631728 2024-11-15T23:17:17,322 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33b0d42b6ce849ecbd43f7d71ff8757c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731712632911 2024-11-15T23:17:17,323 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 195f597632164ca5a6c8a0202eedc709, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731712635180 2024-11-15T23:17:17,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742284_1460 (size=12151) 2024-11-15T23:17:17,359 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#C#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:17,360 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/2eeff63e8d9e4a8dac69caeb6dcdd686 is 50, key is test_row_0/C:col10/1731712635180/Put/seqid=0 2024-11-15T23:17:17,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:17,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742285_1461 (size=12241) 2024-11-15T23:17:17,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712637384, max timestamp=1731712637456Num uniques 859; 2024-11-15T23:17:17,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712637384, max timestamp=1731712637456Num uniques 859; 2024-11-15T23:17:17,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:17:17,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712637384, max timestamp=1731712637456Num uniques 859; 2024-11-15T23:17:17,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:17:17,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:17:17,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T23:17:17,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:17,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712697509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:17,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:17,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:17,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712697522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:17,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712697522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:17,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:17,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:17,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712697524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:17,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:17,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712697526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:17,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:17,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:17,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712697638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:17,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:17,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712697645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:17,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:17,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712697645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:17,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712697657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712697657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:17,715 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/9a88b40ccd0b4694979f9d60ffe47c81 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/9a88b40ccd0b4694979f9d60ffe47c81 2024-11-15T23:17:17,733 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/3a6d4d071a614b05ae4305c7f724a0c6 2024-11-15T23:17:17,735 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/B of 3ad510e3715c6faa652057022b8e1f17 into 9a88b40ccd0b4694979f9d60ffe47c81(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:17,735 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:17,735 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/B, priority=12, startTime=1731712637161; duration=0sec 2024-11-15T23:17:17,735 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:17,735 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:B 2024-11-15T23:17:17,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/de43399a62834383bd579aeb92ecce9f is 50, key is test_row_0/C:col10/1731712636322/Put/seqid=0 2024-11-15T23:17:17,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742286_1462 (size=12151) 2024-11-15T23:17:17,772 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/de43399a62834383bd579aeb92ecce9f 2024-11-15T23:17:17,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/bdabe2ae55c54b8fa027864ae940470c as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/bdabe2ae55c54b8fa027864ae940470c 2024-11-15T23:17:17,783 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/bdabe2ae55c54b8fa027864ae940470c, entries=150, sequenceid=137, filesize=11.9 K 2024-11-15T23:17:17,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/3a6d4d071a614b05ae4305c7f724a0c6 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/3a6d4d071a614b05ae4305c7f724a0c6 2024-11-15T23:17:17,799 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/2eeff63e8d9e4a8dac69caeb6dcdd686 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/2eeff63e8d9e4a8dac69caeb6dcdd686 2024-11-15T23:17:17,799 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/3a6d4d071a614b05ae4305c7f724a0c6, entries=150, sequenceid=137, filesize=11.9 K 2024-11-15T23:17:17,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/de43399a62834383bd579aeb92ecce9f as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/de43399a62834383bd579aeb92ecce9f 2024-11-15T23:17:17,808 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/C of 3ad510e3715c6faa652057022b8e1f17 into 2eeff63e8d9e4a8dac69caeb6dcdd686(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:17,808 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:17,808 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/C, priority=12, startTime=1731712637161; duration=0sec 2024-11-15T23:17:17,808 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:17,808 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:C 2024-11-15T23:17:17,819 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/de43399a62834383bd579aeb92ecce9f, entries=150, sequenceid=137, filesize=11.9 K 2024-11-15T23:17:17,820 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3ad510e3715c6faa652057022b8e1f17 in 647ms, sequenceid=137, compaction requested=false 2024-11-15T23:17:17,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:17,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:17,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-15T23:17:17,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-15T23:17:17,823 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-15T23:17:17,823 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 953 msec 2024-11-15T23:17:17,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 957 msec 2024-11-15T23:17:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:17,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-15T23:17:17,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:17,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:17,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:17,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:17,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:17,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:17,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/aaaa39d856524a68bdf67832bd6b6d82 is 50, key is test_row_0/A:col10/1731712637874/Put/seqid=0 2024-11-15T23:17:17,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712697895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:17,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:17,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712697896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712697897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:17,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712697899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:17,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742287_1463 (size=14541) 2024-11-15T23:17:17,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:17,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712697904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:17,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:17,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T23:17:18,006 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:18,006 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:17:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-15T23:17:18,008 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:17:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T23:17:18,009 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:17:18,009 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:17:18,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712698006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712698025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712698025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712698025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712698035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T23:17:18,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T23:17:18,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:18,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:18,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:18,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:18,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:18,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712698225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712698246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712698246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712698246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712698257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T23:17:18,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:18,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:18,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:18,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:18,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:18,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/aaaa39d856524a68bdf67832bd6b6d82 2024-11-15T23:17:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T23:17:18,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/e1519e402e484ec78b57d96dac898044 is 50, key is test_row_0/B:col10/1731712637874/Put/seqid=0 2024-11-15T23:17:18,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742288_1464 (size=12151) 2024-11-15T23:17:18,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/e1519e402e484ec78b57d96dac898044 2024-11-15T23:17:18,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/813e8f9223434b9fbfbc5fced5edd24e is 50, key is test_row_0/C:col10/1731712637874/Put/seqid=0 2024-11-15T23:17:18,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742289_1465 (size=12151) 2024-11-15T23:17:18,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/813e8f9223434b9fbfbc5fced5edd24e 2024-11-15T23:17:18,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/aaaa39d856524a68bdf67832bd6b6d82 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/aaaa39d856524a68bdf67832bd6b6d82 2024-11-15T23:17:18,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/aaaa39d856524a68bdf67832bd6b6d82, entries=200, sequenceid=177, filesize=14.2 K 2024-11-15T23:17:18,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/e1519e402e484ec78b57d96dac898044 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/e1519e402e484ec78b57d96dac898044 2024-11-15T23:17:18,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/e1519e402e484ec78b57d96dac898044, entries=150, sequenceid=177, filesize=11.9 K 2024-11-15T23:17:18,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/813e8f9223434b9fbfbc5fced5edd24e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/813e8f9223434b9fbfbc5fced5edd24e 2024-11-15T23:17:18,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/813e8f9223434b9fbfbc5fced5edd24e, entries=150, sequenceid=177, filesize=11.9 K 2024-11-15T23:17:18,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 3ad510e3715c6faa652057022b8e1f17 in 537ms, sequenceid=177, compaction requested=true 2024-11-15T23:17:18,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:18,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:17:18,412 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:18,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:18,412 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:18,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:17:18,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:18,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:17:18,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:18,413 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:18,413 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38933 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:18,413 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/A is initiating minor compaction (all files) 2024-11-15T23:17:18,413 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/A in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:18,413 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/f5aee2c6cab148f6afe6cdc782e62bd1, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/bdabe2ae55c54b8fa027864ae940470c, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/aaaa39d856524a68bdf67832bd6b6d82] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=38.0 K 2024-11-15T23:17:18,413 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/B is initiating minor compaction (all files) 2024-11-15T23:17:18,414 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/B in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:18,414 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/9a88b40ccd0b4694979f9d60ffe47c81, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/3a6d4d071a614b05ae4305c7f724a0c6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/e1519e402e484ec78b57d96dac898044] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=35.7 K 2024-11-15T23:17:18,414 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a88b40ccd0b4694979f9d60ffe47c81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731712635180 2024-11-15T23:17:18,414 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5aee2c6cab148f6afe6cdc782e62bd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731712635180 2024-11-15T23:17:18,414 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a6d4d071a614b05ae4305c7f724a0c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731712636322 2024-11-15T23:17:18,414 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdabe2ae55c54b8fa027864ae940470c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731712636322 2024-11-15T23:17:18,415 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaaa39d856524a68bdf67832bd6b6d82, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731712637521 2024-11-15T23:17:18,415 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting e1519e402e484ec78b57d96dac898044, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731712637521 2024-11-15T23:17:18,453 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#A#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:18,453 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/b1fbc4d733a24a979f1754596a46db75 is 50, key is test_row_0/A:col10/1731712637874/Put/seqid=0 2024-11-15T23:17:18,462 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#B#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:18,463 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/a82c0a476a8d46afb3b736e9d759f780 is 50, key is test_row_0/B:col10/1731712637874/Put/seqid=0 2024-11-15T23:17:18,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T23:17:18,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:18,468 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-15T23:17:18,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:18,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:18,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:18,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:18,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:18,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:18,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742290_1466 (size=12493) 2024-11-15T23:17:18,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/1844a16f41734560b3f0910ca0cc7ead is 50, key is test_row_0/A:col10/1731712637876/Put/seqid=0 2024-11-15T23:17:18,487 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/b1fbc4d733a24a979f1754596a46db75 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/b1fbc4d733a24a979f1754596a46db75 2024-11-15T23:17:18,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742291_1467 (size=12493) 2024-11-15T23:17:18,492 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/A of 3ad510e3715c6faa652057022b8e1f17 into b1fbc4d733a24a979f1754596a46db75(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:18,492 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:18,492 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/A, priority=13, startTime=1731712638412; duration=0sec 2024-11-15T23:17:18,492 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:18,492 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:A 2024-11-15T23:17:18,492 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:18,493 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:18,493 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/C is initiating minor compaction (all files) 2024-11-15T23:17:18,493 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/C in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:18,493 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/2eeff63e8d9e4a8dac69caeb6dcdd686, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/de43399a62834383bd579aeb92ecce9f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/813e8f9223434b9fbfbc5fced5edd24e] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=35.7 K 2024-11-15T23:17:18,494 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eeff63e8d9e4a8dac69caeb6dcdd686, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731712635180 2024-11-15T23:17:18,496 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/a82c0a476a8d46afb3b736e9d759f780 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/a82c0a476a8d46afb3b736e9d759f780 2024-11-15T23:17:18,496 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting de43399a62834383bd579aeb92ecce9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731712636322 2024-11-15T23:17:18,499 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 813e8f9223434b9fbfbc5fced5edd24e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731712637521 2024-11-15T23:17:18,509 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/B of 3ad510e3715c6faa652057022b8e1f17 into a82c0a476a8d46afb3b736e9d759f780(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:18,509 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:18,510 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/B, priority=13, startTime=1731712638412; duration=0sec 2024-11-15T23:17:18,510 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:18,510 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:B 2024-11-15T23:17:18,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742292_1468 (size=12151) 2024-11-15T23:17:18,510 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#C#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:18,511 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/7c1ac42eea6646a59e16a8bfe06f4c49 is 50, key is test_row_0/C:col10/1731712637874/Put/seqid=0 2024-11-15T23:17:18,512 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/1844a16f41734560b3f0910ca0cc7ead 2024-11-15T23:17:18,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/c44686efd6ae4434bf4ede92a12d0aac is 50, key is test_row_0/B:col10/1731712637876/Put/seqid=0 2024-11-15T23:17:18,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742293_1469 (size=12493) 2024-11-15T23:17:18,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742294_1470 (size=12151) 2024-11-15T23:17:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:18,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:18,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712638585, max timestamp=1731712638602Num uniques 859; 2024-11-15T23:17:18,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:17:18,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712638585, max timestamp=1731712638602Num uniques 859; 2024-11-15T23:17:18,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:17:18,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712638585, max timestamp=1731712638602Num uniques 859; 2024-11-15T23:17:18,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:17:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T23:17:18,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712698635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712698636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712698637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712698650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712698653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712698756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712698756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712698756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712698775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712698776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,941 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/7c1ac42eea6646a59e16a8bfe06f4c49 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/7c1ac42eea6646a59e16a8bfe06f4c49 2024-11-15T23:17:18,945 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/C of 3ad510e3715c6faa652057022b8e1f17 into 7c1ac42eea6646a59e16a8bfe06f4c49(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:18,945 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:18,945 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/C, priority=13, startTime=1731712638412; duration=0sec 2024-11-15T23:17:18,945 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:18,945 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:C 2024-11-15T23:17:18,948 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/c44686efd6ae4434bf4ede92a12d0aac 2024-11-15T23:17:18,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/cfb89b421f34406aaf2fe2fba4d96772 is 50, key is test_row_0/C:col10/1731712637876/Put/seqid=0 2024-11-15T23:17:18,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742295_1471 (size=12151) 2024-11-15T23:17:18,969 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/cfb89b421f34406aaf2fe2fba4d96772 2024-11-15T23:17:18,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/1844a16f41734560b3f0910ca0cc7ead as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/1844a16f41734560b3f0910ca0cc7ead 2024-11-15T23:17:18,977 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/1844a16f41734560b3f0910ca0cc7ead, entries=150, sequenceid=186, filesize=11.9 K 2024-11-15T23:17:18,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712698965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712698965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/c44686efd6ae4434bf4ede92a12d0aac as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/c44686efd6ae4434bf4ede92a12d0aac 2024-11-15T23:17:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712698965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:18,982 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/c44686efd6ae4434bf4ede92a12d0aac, entries=150, sequenceid=186, filesize=11.9 K 2024-11-15T23:17:18,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/cfb89b421f34406aaf2fe2fba4d96772 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/cfb89b421f34406aaf2fe2fba4d96772 2024-11-15T23:17:18,986 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/cfb89b421f34406aaf2fe2fba4d96772, entries=150, sequenceid=186, filesize=11.9 K 2024-11-15T23:17:18,986 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 3ad510e3715c6faa652057022b8e1f17 in 518ms, sequenceid=186, compaction requested=false 2024-11-15T23:17:18,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:18,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:18,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-15T23:17:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-15T23:17:18,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-15T23:17:18,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 979 msec 2024-11-15T23:17:18,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 983 msec 2024-11-15T23:17:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:19,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-15T23:17:19,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731712638633, max timestamp=1731712639001Num uniques 1068; 2024-11-15T23:17:19,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731712638633, max timestamp=1731712639001Num uniques 1068; 2024-11-15T23:17:19,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-15T23:17:19,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731712638633, max timestamp=1731712639001Num uniques 1068; 2024-11-15T23:17:19,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-15T23:17:19,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-15T23:17:19,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:19,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:19,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:19,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:19,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:19,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:19,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/6083af8dcaf44b7b808559785f89029c is 50, key is test_row_0/A:col10/1731712639001/Put/seqid=0 2024-11-15T23:17:19,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742296_1472 (size=16931) 2024-11-15T23:17:19,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712639079, max timestamp=1731712639112Num uniques 859; 2024-11-15T23:17:19,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712639079, max timestamp=1731712639112Num uniques 859; 2024-11-15T23:17:19,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:17:19,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:17:19,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712639079, max timestamp=1731712639112Num uniques 859; 2024-11-15T23:17:19,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:17:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T23:17:19,148 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:19,148 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:17:19,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-15T23:17:19,150 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:17:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T23:17:19,150 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:17:19,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:17:19,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731712639133, max timestamp=1731712639234Num uniques 1068; 2024-11-15T23:17:19,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731712639133, max timestamp=1731712639234Num uniques 1068; 2024-11-15T23:17:19,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-15T23:17:19,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-15T23:17:19,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731712639133, max timestamp=1731712639234Num uniques 1068; 2024-11-15T23:17:19,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-15T23:17:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T23:17:19,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T23:17:19,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:19,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:19,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:19,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:19,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:19,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:19,326 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731712639274, max timestamp=1731712639321Num uniques 1077; 2024-11-15T23:17:19,326 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-15T23:17:19,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731712639307, max timestamp=1731712639321Num uniques 1077; 2024-11-15T23:17:19,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=6870, suffix heap size=139976,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=150 2024-11-15T23:17:19,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731712639274, max timestamp=1731712639321Num uniques 1077; 2024-11-15T23:17:19,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-15T23:17:19,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1731712639346, max timestamp=1731712639400Num uniques 1136; 2024-11-15T23:17:19,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=17676, suffix heap size=146112,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=386 2024-11-15T23:17:19,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731712639346, max timestamp=1731712639400Num uniques 1136; 2024-11-15T23:17:19,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-15T23:17:19,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731712639346, max timestamp=1731712639400Num uniques 1009; 2024-11-15T23:17:19,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10806, suffix heap size=132904,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=236 2024-11-15T23:17:19,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/6083af8dcaf44b7b808559785f89029c 2024-11-15T23:17:19,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/7374efeafa864390aa5500d6d151d939 is 50, key is test_row_0/B:col10/1731712639001/Put/seqid=0 2024-11-15T23:17:19,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742297_1473 (size=12151) 2024-11-15T23:17:19,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T23:17:19,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:19,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:19,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:19,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:19,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T23:17:19,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712699451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:19,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731712639424, max timestamp=1731712639473Num uniques 1095; 2024-11-15T23:17:19,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-15T23:17:19,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731712639424, max timestamp=1731712639473Num uniques 1245; 2024-11-15T23:17:19,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=57020, new segment data size=13510, suffix heap size=157448,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1245, new segment cells count=295 2024-11-15T23:17:19,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731712639424, max timestamp=1731712639473Num uniques 1095; 2024-11-15T23:17:19,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-15T23:17:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:19,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:19,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712699542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:19,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:19,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712699542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:19,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712699543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:19,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712699555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:19,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:19,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712699585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:19,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T23:17:19,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:19,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:19,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:19,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:19,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:19,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:19,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712699666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:19,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712699666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712699666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:19,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712699666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:19,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T23:17:19,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:19,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:19,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:19,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:19,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:19,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T23:17:19,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:19,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712699805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:19,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:19,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/7374efeafa864390aa5500d6d151d939 2024-11-15T23:17:19,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/4f252661893343c689f429f7a20f0c14 is 50, key is test_row_0/C:col10/1731712639001/Put/seqid=0 2024-11-15T23:17:19,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742298_1474 (size=12151) 2024-11-15T23:17:19,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:19,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712699875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:19,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:19,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:19,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712699875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:19,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:19,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712699875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:19,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:19,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712699885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:19,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:19,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:19,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T23:17:19,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:19,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:19,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:19,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:19,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:19,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:20,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T23:17:20,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:20,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:20,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:20,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:20,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:20,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:20,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712700125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:20,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:20,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:20,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712700185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:20,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712700185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:20,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712700185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:20,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T23:17:20,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:20,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:20,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:20,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:20,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:20,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712700215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:20,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/4f252661893343c689f429f7a20f0c14 2024-11-15T23:17:20,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/6083af8dcaf44b7b808559785f89029c as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/6083af8dcaf44b7b808559785f89029c 2024-11-15T23:17:20,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/6083af8dcaf44b7b808559785f89029c, entries=250, sequenceid=228, filesize=16.5 K 2024-11-15T23:17:20,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/7374efeafa864390aa5500d6d151d939 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/7374efeafa864390aa5500d6d151d939 2024-11-15T23:17:20,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/7374efeafa864390aa5500d6d151d939, entries=150, sequenceid=228, filesize=11.9 K 2024-11-15T23:17:20,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/4f252661893343c689f429f7a20f0c14 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4f252661893343c689f429f7a20f0c14 2024-11-15T23:17:20,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4f252661893343c689f429f7a20f0c14, entries=150, sequenceid=228, filesize=11.9 K 2024-11-15T23:17:20,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=154.31 KB/158010 for 3ad510e3715c6faa652057022b8e1f17 in 1251ms, sequenceid=228, compaction requested=true 2024-11-15T23:17:20,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:20,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:17:20,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:20,255 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:20,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:17:20,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:20,256 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:20,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:17:20,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:20,256 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:20,256 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:20,256 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/A is initiating minor compaction (all files) 2024-11-15T23:17:20,256 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/B is initiating minor compaction (all files) 2024-11-15T23:17:20,256 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/A in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:20,256 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/B in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:20,256 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/a82c0a476a8d46afb3b736e9d759f780, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/c44686efd6ae4434bf4ede92a12d0aac, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/7374efeafa864390aa5500d6d151d939] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=35.9 K 2024-11-15T23:17:20,256 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/b1fbc4d733a24a979f1754596a46db75, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/1844a16f41734560b3f0910ca0cc7ead, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/6083af8dcaf44b7b808559785f89029c] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=40.6 K 2024-11-15T23:17:20,257 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1fbc4d733a24a979f1754596a46db75, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731712637521 2024-11-15T23:17:20,257 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting a82c0a476a8d46afb3b736e9d759f780, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731712637521 2024-11-15T23:17:20,259 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1844a16f41734560b3f0910ca0cc7ead, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731712637876 2024-11-15T23:17:20,259 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting c44686efd6ae4434bf4ede92a12d0aac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731712637876 2024-11-15T23:17:20,259 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 7374efeafa864390aa5500d6d151d939, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731712638633 2024-11-15T23:17:20,259 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6083af8dcaf44b7b808559785f89029c, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731712638633 2024-11-15T23:17:20,265 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#B#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:20,265 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/17b14d5f314f45939f5536b6a3b47b03 is 50, key is test_row_0/B:col10/1731712639001/Put/seqid=0 2024-11-15T23:17:20,266 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#A#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:20,267 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/631352e365e14997a680804ff16cccd7 is 50, key is test_row_0/A:col10/1731712639001/Put/seqid=0 2024-11-15T23:17:20,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742299_1475 (size=12595) 2024-11-15T23:17:20,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742300_1476 (size=12595) 2024-11-15T23:17:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T23:17:20,293 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/17b14d5f314f45939f5536b6a3b47b03 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/17b14d5f314f45939f5536b6a3b47b03 2024-11-15T23:17:20,297 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/B of 3ad510e3715c6faa652057022b8e1f17 into 17b14d5f314f45939f5536b6a3b47b03(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:20,297 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:20,297 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/B, priority=13, startTime=1731712640255; duration=0sec 2024-11-15T23:17:20,297 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:20,297 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:B 2024-11-15T23:17:20,297 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:20,298 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:20,298 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/C is initiating minor compaction (all files) 2024-11-15T23:17:20,298 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/C in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:20,298 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/7c1ac42eea6646a59e16a8bfe06f4c49, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/cfb89b421f34406aaf2fe2fba4d96772, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4f252661893343c689f429f7a20f0c14] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=35.9 K 2024-11-15T23:17:20,298 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c1ac42eea6646a59e16a8bfe06f4c49, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731712637521 2024-11-15T23:17:20,298 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting cfb89b421f34406aaf2fe2fba4d96772, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731712637876 2024-11-15T23:17:20,300 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f252661893343c689f429f7a20f0c14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731712638633 2024-11-15T23:17:20,306 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#C#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:20,306 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/b456fc1d0e18469fa1b4a26eb3bdea8b is 50, key is test_row_0/C:col10/1731712639001/Put/seqid=0 2024-11-15T23:17:20,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742301_1477 (size=12595) 2024-11-15T23:17:20,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T23:17:20,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:20,376 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=154.31 KB heapSize=432 KB 2024-11-15T23:17:20,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:20,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:20,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:20,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:20,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:20,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:20,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/61677d84073c4c97bdb0f3391e1c6837 is 50, key is test_row_0/A:col10/1731712639553/Put/seqid=0 2024-11-15T23:17:20,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742302_1478 (size=12301) 2024-11-15T23:17:20,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:20,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:20,675 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/631352e365e14997a680804ff16cccd7 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/631352e365e14997a680804ff16cccd7 2024-11-15T23:17:20,679 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/A of 3ad510e3715c6faa652057022b8e1f17 into 631352e365e14997a680804ff16cccd7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:20,679 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:20,679 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/A, priority=13, startTime=1731712640255; duration=0sec 2024-11-15T23:17:20,679 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:20,680 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:A 2024-11-15T23:17:20,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712700688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:20,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:20,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712700705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:20,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712700706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:20,716 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/b456fc1d0e18469fa1b4a26eb3bdea8b as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/b456fc1d0e18469fa1b4a26eb3bdea8b 2024-11-15T23:17:20,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712700707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:20,727 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/C of 3ad510e3715c6faa652057022b8e1f17 into b456fc1d0e18469fa1b4a26eb3bdea8b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:20,727 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:20,727 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/C, priority=13, startTime=1731712640256; duration=0sec 2024-11-15T23:17:20,727 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:20,727 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:C 2024-11-15T23:17:20,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:20,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712700735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:20,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:20,794 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/61677d84073c4c97bdb0f3391e1c6837 2024-11-15T23:17:20,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/edd2ad7fada245cbaf12daa3f193dca6 is 50, key is test_row_0/B:col10/1731712639553/Put/seqid=0 2024-11-15T23:17:20,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742303_1479 (size=12301) 2024-11-15T23:17:20,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:20,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712700815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:20,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:20,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:21,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712701025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:21,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:21,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:21,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:21,213 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/edd2ad7fada245cbaf12daa3f193dca6 2024-11-15T23:17:21,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/9ba42adc39fc4c738f0e7fdb6ff98f0b is 50, key is test_row_0/C:col10/1731712639553/Put/seqid=0 2024-11-15T23:17:21,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742304_1480 (size=12301) 2024-11-15T23:17:21,237 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/9ba42adc39fc4c738f0e7fdb6ff98f0b 2024-11-15T23:17:21,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/61677d84073c4c97bdb0f3391e1c6837 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/61677d84073c4c97bdb0f3391e1c6837 2024-11-15T23:17:21,242 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/61677d84073c4c97bdb0f3391e1c6837, entries=150, sequenceid=335, filesize=12.0 K 2024-11-15T23:17:21,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/edd2ad7fada245cbaf12daa3f193dca6 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/edd2ad7fada245cbaf12daa3f193dca6 2024-11-15T23:17:21,247 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/edd2ad7fada245cbaf12daa3f193dca6, entries=150, sequenceid=335, filesize=12.0 K 2024-11-15T23:17:21,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/9ba42adc39fc4c738f0e7fdb6ff98f0b as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/9ba42adc39fc4c738f0e7fdb6ff98f0b 2024-11-15T23:17:21,251 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/9ba42adc39fc4c738f0e7fdb6ff98f0b, entries=150, sequenceid=335, filesize=12.0 K 2024-11-15T23:17:21,252 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~431.95 KB/442320, currentSize=26.84 KB/27480 for 3ad510e3715c6faa652057022b8e1f17 in 876ms, sequenceid=335, compaction requested=false 2024-11-15T23:17:21,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:21,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:21,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-15T23:17:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-15T23:17:21,257 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-15T23:17:21,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1050 sec 2024-11-15T23:17:21,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.1100 sec 2024-11-15T23:17:21,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T23:17:21,306 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:21,307 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:17:21,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-15T23:17:21,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T23:17:21,309 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:17:21,310 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:17:21,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:17:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:21,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T23:17:21,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:21,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:21,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:21,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:21,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:21,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:21,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/47d2092ccea444e482e577204e1c8ed9 is 50, key is test_row_0/A:col10/1731712641347/Put/seqid=0 2024-11-15T23:17:21,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742305_1481 (size=17181) 2024-11-15T23:17:21,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/47d2092ccea444e482e577204e1c8ed9 2024-11-15T23:17:21,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/ec616f00cc2642dbaceb81a961664867 is 50, key is test_row_0/B:col10/1731712641347/Put/seqid=0 2024-11-15T23:17:21,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742306_1482 (size=12301) 2024-11-15T23:17:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T23:17:21,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T23:17:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:21,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:21,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712641481, max timestamp=1731712641518Num uniques 859; 2024-11-15T23:17:21,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:17:21,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712641481, max timestamp=1731712641518Num uniques 859; 2024-11-15T23:17:21,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712641481, max timestamp=1731712641518Num uniques 859; 2024-11-15T23:17:21,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:17:21,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:17:21,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T23:17:21,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:21,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:21,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:21,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:21,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T23:17:21,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:21,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712701693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:21,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:21,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:21,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:21,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712701715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:21,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712701715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:21,724 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in 3ad510e3715c6faa652057022b8e1f17 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:17:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:21,724 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in 3ad510e3715c6faa652057022b8e1f17 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:17:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:21,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712701735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:21,749 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_1' in 3ad510e3715c6faa652057022b8e1f17 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:17:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:21,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712701756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:21,767 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_0' in 3ad510e3715c6faa652057022b8e1f17 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:17:21,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T23:17:21,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:21,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:21,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:21,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:21,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:21,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:21,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:21,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:21,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/ec616f00cc2642dbaceb81a961664867 2024-11-15T23:17:21,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/227c0203ab8241ca9ddbd4d8e7958f66 is 50, key is test_row_0/C:col10/1731712641347/Put/seqid=0 2024-11-15T23:17:21,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712701805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:21,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742307_1483 (size=12301) 2024-11-15T23:17:21,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/227c0203ab8241ca9ddbd4d8e7958f66 2024-11-15T23:17:21,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/47d2092ccea444e482e577204e1c8ed9 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/47d2092ccea444e482e577204e1c8ed9 2024-11-15T23:17:21,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/47d2092ccea444e482e577204e1c8ed9, entries=250, sequenceid=347, filesize=16.8 K 2024-11-15T23:17:21,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/ec616f00cc2642dbaceb81a961664867 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/ec616f00cc2642dbaceb81a961664867 2024-11-15T23:17:21,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/ec616f00cc2642dbaceb81a961664867, entries=150, sequenceid=347, filesize=12.0 K 2024-11-15T23:17:21,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/227c0203ab8241ca9ddbd4d8e7958f66 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/227c0203ab8241ca9ddbd4d8e7958f66 2024-11-15T23:17:21,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/227c0203ab8241ca9ddbd4d8e7958f66, entries=150, sequenceid=347, filesize=12.0 K 2024-11-15T23:17:21,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3ad510e3715c6faa652057022b8e1f17 in 507ms, sequenceid=347, compaction requested=true 2024-11-15T23:17:21,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:21,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:17:21,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:21,856 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:21,856 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:21,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:17:21,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:21,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:17:21,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:21,857 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:21,858 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/B is initiating minor compaction (all files) 2024-11-15T23:17:21,858 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/B in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:21,858 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/17b14d5f314f45939f5536b6a3b47b03, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/edd2ad7fada245cbaf12daa3f193dca6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/ec616f00cc2642dbaceb81a961664867] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=36.3 K 2024-11-15T23:17:21,858 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42077 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:21,858 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/A is initiating minor compaction (all files) 2024-11-15T23:17:21,858 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/A in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:21,858 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/631352e365e14997a680804ff16cccd7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/61677d84073c4c97bdb0f3391e1c6837, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/47d2092ccea444e482e577204e1c8ed9] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=41.1 K 2024-11-15T23:17:21,858 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 17b14d5f314f45939f5536b6a3b47b03, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731712638633 2024-11-15T23:17:21,859 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 631352e365e14997a680804ff16cccd7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731712638633 2024-11-15T23:17:21,859 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting edd2ad7fada245cbaf12daa3f193dca6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731712639516 2024-11-15T23:17:21,859 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61677d84073c4c97bdb0f3391e1c6837, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731712639516 2024-11-15T23:17:21,859 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting ec616f00cc2642dbaceb81a961664867, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731712640666 2024-11-15T23:17:21,859 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47d2092ccea444e482e577204e1c8ed9, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731712640666 2024-11-15T23:17:21,866 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#A#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:21,867 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/5312aa893b2e47ed9a50da3c15e8d31a is 50, key is test_row_0/A:col10/1731712641347/Put/seqid=0 2024-11-15T23:17:21,874 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#B#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:21,874 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/95a93a867ca744a1b410b9cbaeeb09ff is 50, key is test_row_0/B:col10/1731712641347/Put/seqid=0 2024-11-15T23:17:21,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742309_1485 (size=12847) 2024-11-15T23:17:21,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T23:17:21,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:21,922 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-15T23:17:21,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:21,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:21,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:21,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:21,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:21,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:21,926 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/95a93a867ca744a1b410b9cbaeeb09ff as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/95a93a867ca744a1b410b9cbaeeb09ff 2024-11-15T23:17:21,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742308_1484 (size=12847) 2024-11-15T23:17:21,935 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/B of 3ad510e3715c6faa652057022b8e1f17 into 95a93a867ca744a1b410b9cbaeeb09ff(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:21,935 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:21,935 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/B, priority=13, startTime=1731712641856; duration=0sec 2024-11-15T23:17:21,935 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:21,935 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:B 2024-11-15T23:17:21,935 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T23:17:21,936 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:21,936 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/C is initiating minor compaction (all files) 2024-11-15T23:17:21,937 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/C in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:21,937 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/b456fc1d0e18469fa1b4a26eb3bdea8b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/9ba42adc39fc4c738f0e7fdb6ff98f0b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/227c0203ab8241ca9ddbd4d8e7958f66] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=36.3 K 2024-11-15T23:17:21,937 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting b456fc1d0e18469fa1b4a26eb3bdea8b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731712638633 2024-11-15T23:17:21,938 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ba42adc39fc4c738f0e7fdb6ff98f0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731712639516 2024-11-15T23:17:21,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/f6b1be0ff4d44482b3a66a3d08b1e598 is 50, key is test_row_0/A:col10/1731712641678/Put/seqid=0 2024-11-15T23:17:21,945 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 227c0203ab8241ca9ddbd4d8e7958f66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731712640666 2024-11-15T23:17:21,959 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#C#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:21,960 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/685ac11ecfb143cd9ec9aaa3891911a5 is 50, key is test_row_0/C:col10/1731712641347/Put/seqid=0 2024-11-15T23:17:21,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742310_1486 (size=12301) 2024-11-15T23:17:21,967 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/f6b1be0ff4d44482b3a66a3d08b1e598 2024-11-15T23:17:21,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742311_1487 (size=12847) 2024-11-15T23:17:21,977 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/685ac11ecfb143cd9ec9aaa3891911a5 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/685ac11ecfb143cd9ec9aaa3891911a5 2024-11-15T23:17:21,982 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/C of 3ad510e3715c6faa652057022b8e1f17 into 685ac11ecfb143cd9ec9aaa3891911a5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:21,982 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:21,982 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/C, priority=13, startTime=1731712641857; duration=0sec 2024-11-15T23:17:21,982 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:21,982 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:C 2024-11-15T23:17:21,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/91d42e86823b4339a1161ce121b26b09 is 50, key is test_row_0/B:col10/1731712641678/Put/seqid=0 2024-11-15T23:17:22,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742312_1488 (size=12301) 2024-11-15T23:17:22,004 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/91d42e86823b4339a1161ce121b26b09 2024-11-15T23:17:22,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/b29ca0bb3a024cd49619d7ea6f36a40e is 50, key is test_row_0/C:col10/1731712641678/Put/seqid=0 2024-11-15T23:17:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:22,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:22,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742313_1489 (size=12301) 2024-11-15T23:17:22,039 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/b29ca0bb3a024cd49619d7ea6f36a40e 2024-11-15T23:17:22,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/f6b1be0ff4d44482b3a66a3d08b1e598 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/f6b1be0ff4d44482b3a66a3d08b1e598 2024-11-15T23:17:22,051 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/f6b1be0ff4d44482b3a66a3d08b1e598, entries=150, sequenceid=384, filesize=12.0 K 2024-11-15T23:17:22,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/91d42e86823b4339a1161ce121b26b09 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/91d42e86823b4339a1161ce121b26b09 2024-11-15T23:17:22,088 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/91d42e86823b4339a1161ce121b26b09, entries=150, sequenceid=384, filesize=12.0 K 2024-11-15T23:17:22,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/b29ca0bb3a024cd49619d7ea6f36a40e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/b29ca0bb3a024cd49619d7ea6f36a40e 2024-11-15T23:17:22,100 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/b29ca0bb3a024cd49619d7ea6f36a40e, entries=150, sequenceid=384, filesize=12.0 K 2024-11-15T23:17:22,103 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=13.42 KB/13740 for 3ad510e3715c6faa652057022b8e1f17 in 181ms, sequenceid=384, compaction requested=false 2024-11-15T23:17:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-15T23:17:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-15T23:17:22,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-15T23:17:22,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 799 msec 2024-11-15T23:17:22,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 806 msec 2024-11-15T23:17:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:22,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T23:17:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:22,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/b76d1701b20746be9ff20f06c0b81617 is 50, key is test_row_0/A:col10/1731712642118/Put/seqid=0 2024-11-15T23:17:22,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742314_1490 (size=14741) 2024-11-15T23:17:22,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712642225, max timestamp=1731712642240Num uniques 859; 2024-11-15T23:17:22,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:17:22,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712642225, max timestamp=1731712642240Num uniques 859; 2024-11-15T23:17:22,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:17:22,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712642225, max timestamp=1731712642240Num uniques 859; 2024-11-15T23:17:22,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:17:22,331 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/5312aa893b2e47ed9a50da3c15e8d31a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/5312aa893b2e47ed9a50da3c15e8d31a 2024-11-15T23:17:22,334 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/A of 3ad510e3715c6faa652057022b8e1f17 into 5312aa893b2e47ed9a50da3c15e8d31a(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:22,335 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:22,335 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/A, priority=13, startTime=1731712641856; duration=0sec 2024-11-15T23:17:22,335 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:22,335 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:A 2024-11-15T23:17:22,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712702370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:22,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:22,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:22,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T23:17:22,446 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:22,446 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:17:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-15T23:17:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T23:17:22,448 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:17:22,448 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:17:22,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:17:22,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712702485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:22,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/b76d1701b20746be9ff20f06c0b81617 2024-11-15T23:17:22,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/8b3cec4203104e6fa3b7448c2f16e8bc is 50, key is test_row_0/B:col10/1731712642118/Put/seqid=0 2024-11-15T23:17:22,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742315_1491 (size=12301) 2024-11-15T23:17:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T23:17:22,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T23:17:22,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:22,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:22,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:22,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:22,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:22,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712702695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:22,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T23:17:22,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:22,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:22,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:22,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:22,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T23:17:22,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T23:17:22,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:22,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:22,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:22,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:22,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:22,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/8b3cec4203104e6fa3b7448c2f16e8bc 2024-11-15T23:17:22,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/4aa4f045a9cb46ceace51ccab0fddab6 is 50, key is test_row_0/C:col10/1731712642118/Put/seqid=0 2024-11-15T23:17:22,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742316_1492 (size=12301) 2024-11-15T23:17:23,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712703005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:23,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T23:17:23,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:23,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:23,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:23,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:23,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T23:17:23,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T23:17:23,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:23,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:23,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:23,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:23,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:23,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/4aa4f045a9cb46ceace51ccab0fddab6 2024-11-15T23:17:23,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/b76d1701b20746be9ff20f06c0b81617 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/b76d1701b20746be9ff20f06c0b81617 2024-11-15T23:17:23,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T23:17:23,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:23,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:23,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:23,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:23,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:23,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/b76d1701b20746be9ff20f06c0b81617, entries=200, sequenceid=396, filesize=14.4 K 2024-11-15T23:17:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/8b3cec4203104e6fa3b7448c2f16e8bc as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/8b3cec4203104e6fa3b7448c2f16e8bc 2024-11-15T23:17:23,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/8b3cec4203104e6fa3b7448c2f16e8bc, entries=150, sequenceid=396, filesize=12.0 K 2024-11-15T23:17:23,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/4aa4f045a9cb46ceace51ccab0fddab6 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4aa4f045a9cb46ceace51ccab0fddab6 2024-11-15T23:17:23,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4aa4f045a9cb46ceace51ccab0fddab6, entries=150, sequenceid=396, filesize=12.0 K 2024-11-15T23:17:23,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3ad510e3715c6faa652057022b8e1f17 in 1277ms, sequenceid=396, compaction requested=true 2024-11-15T23:17:23,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:23,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:17:23,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:23,398 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:23,398 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:23,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:17:23,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:23,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:17:23,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:23,399 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:23,399 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:23,399 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/A is initiating minor compaction (all files) 2024-11-15T23:17:23,399 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/B is initiating minor compaction (all files) 2024-11-15T23:17:23,399 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/B in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:23,399 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/A in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:23,399 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/95a93a867ca744a1b410b9cbaeeb09ff, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/91d42e86823b4339a1161ce121b26b09, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/8b3cec4203104e6fa3b7448c2f16e8bc] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=36.6 K 2024-11-15T23:17:23,399 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/5312aa893b2e47ed9a50da3c15e8d31a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/f6b1be0ff4d44482b3a66a3d08b1e598, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/b76d1701b20746be9ff20f06c0b81617] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=39.0 K 2024-11-15T23:17:23,399 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 95a93a867ca744a1b410b9cbaeeb09ff, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731712640666 2024-11-15T23:17:23,399 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5312aa893b2e47ed9a50da3c15e8d31a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731712640666 2024-11-15T23:17:23,400 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 91d42e86823b4339a1161ce121b26b09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731712641668 2024-11-15T23:17:23,400 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6b1be0ff4d44482b3a66a3d08b1e598, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731712641668 2024-11-15T23:17:23,400 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b3cec4203104e6fa3b7448c2f16e8bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731712642086 2024-11-15T23:17:23,400 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting b76d1701b20746be9ff20f06c0b81617, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731712642086 2024-11-15T23:17:23,414 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#B#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:23,414 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#A#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:23,415 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/5f486f0664a945dc98001550a96adfbf is 50, key is test_row_0/B:col10/1731712642118/Put/seqid=0 2024-11-15T23:17:23,418 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/3b4d6e1301e54b199c021acf18b715b2 is 50, key is test_row_0/A:col10/1731712642118/Put/seqid=0 2024-11-15T23:17:23,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742317_1493 (size=12949) 2024-11-15T23:17:23,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742318_1494 (size=12949) 2024-11-15T23:17:23,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T23:17:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:23,520 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-15T23:17:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:23,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:23,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/36078ede98c447678675b60727bd0f93 is 50, key is test_row_0/A:col10/1731712642369/Put/seqid=0 2024-11-15T23:17:23,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742319_1495 (size=12301) 2024-11-15T23:17:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:23,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T23:17:23,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712703601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:23,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:23,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:23,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:23,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712703715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:23,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712703735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:23,743 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in 3ad510e3715c6faa652057022b8e1f17 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:17:23,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:23,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:23,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:23,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712703745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:23,755 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in 3ad510e3715c6faa652057022b8e1f17 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:17:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:23,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712703775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:23,783 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_1' in 3ad510e3715c6faa652057022b8e1f17 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:17:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:23,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712703785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:23,795 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_0' in 3ad510e3715c6faa652057022b8e1f17 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:17:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:23,826 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/5f486f0664a945dc98001550a96adfbf as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/5f486f0664a945dc98001550a96adfbf 2024-11-15T23:17:23,829 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/B of 3ad510e3715c6faa652057022b8e1f17 into 5f486f0664a945dc98001550a96adfbf(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:23,829 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:23,829 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/B, priority=13, startTime=1731712643398; duration=0sec 2024-11-15T23:17:23,829 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:23,829 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:B 2024-11-15T23:17:23,830 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:23,830 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:23,830 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/C is initiating minor compaction (all files) 2024-11-15T23:17:23,830 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/C in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:23,830 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/685ac11ecfb143cd9ec9aaa3891911a5, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/b29ca0bb3a024cd49619d7ea6f36a40e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4aa4f045a9cb46ceace51ccab0fddab6] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=36.6 K 2024-11-15T23:17:23,830 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 685ac11ecfb143cd9ec9aaa3891911a5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731712640666 2024-11-15T23:17:23,831 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting b29ca0bb3a024cd49619d7ea6f36a40e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731712641668 2024-11-15T23:17:23,831 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aa4f045a9cb46ceace51ccab0fddab6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731712642086 2024-11-15T23:17:23,836 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#C#compaction#422 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:23,837 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/3b4d6e1301e54b199c021acf18b715b2 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/3b4d6e1301e54b199c021acf18b715b2 2024-11-15T23:17:23,837 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/e962a0847d184ec6b8983d96b06e2ef3 is 50, key is test_row_0/C:col10/1731712642118/Put/seqid=0 2024-11-15T23:17:23,840 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/A of 3ad510e3715c6faa652057022b8e1f17 into 3b4d6e1301e54b199c021acf18b715b2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:23,840 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:23,840 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/A, priority=13, startTime=1731712643398; duration=0sec 2024-11-15T23:17:23,840 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:23,840 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:A 2024-11-15T23:17:23,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742320_1496 (size=12949) 2024-11-15T23:17:23,928 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/36078ede98c447678675b60727bd0f93 2024-11-15T23:17:23,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712703925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:23,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:23,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:23,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:23,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/4a11976230e641a0a6dc69bc79c0fd30 is 50, key is test_row_0/B:col10/1731712642369/Put/seqid=0 2024-11-15T23:17:23,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742321_1497 (size=12301) 2024-11-15T23:17:24,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:24,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712704235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:24,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:24,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:24,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:24,251 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/e962a0847d184ec6b8983d96b06e2ef3 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/e962a0847d184ec6b8983d96b06e2ef3 2024-11-15T23:17:24,254 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/C of 3ad510e3715c6faa652057022b8e1f17 into e962a0847d184ec6b8983d96b06e2ef3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:24,254 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:24,255 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/C, priority=13, startTime=1731712643398; duration=0sec 2024-11-15T23:17:24,255 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:24,255 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:C 2024-11-15T23:17:24,339 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/4a11976230e641a0a6dc69bc79c0fd30 2024-11-15T23:17:24,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/97d3fdbbc7dd4ef89c119dd2d4395127 is 50, key is test_row_0/C:col10/1731712642369/Put/seqid=0 2024-11-15T23:17:24,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742322_1498 (size=12301) 2024-11-15T23:17:24,356 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/97d3fdbbc7dd4ef89c119dd2d4395127 2024-11-15T23:17:24,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/36078ede98c447678675b60727bd0f93 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/36078ede98c447678675b60727bd0f93 2024-11-15T23:17:24,363 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/36078ede98c447678675b60727bd0f93, entries=150, sequenceid=434, filesize=12.0 K 2024-11-15T23:17:24,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/4a11976230e641a0a6dc69bc79c0fd30 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/4a11976230e641a0a6dc69bc79c0fd30 2024-11-15T23:17:24,367 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/4a11976230e641a0a6dc69bc79c0fd30, entries=150, sequenceid=434, filesize=12.0 K 2024-11-15T23:17:24,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/97d3fdbbc7dd4ef89c119dd2d4395127 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/97d3fdbbc7dd4ef89c119dd2d4395127 2024-11-15T23:17:24,374 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/97d3fdbbc7dd4ef89c119dd2d4395127, entries=150, sequenceid=434, filesize=12.0 K 2024-11-15T23:17:24,375 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 3ad510e3715c6faa652057022b8e1f17 in 855ms, sequenceid=434, compaction requested=false 2024-11-15T23:17:24,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:24,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:24,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-15T23:17:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-15T23:17:24,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-15T23:17:24,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9290 sec 2024-11-15T23:17:24,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.9330 sec 2024-11-15T23:17:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T23:17:24,595 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:24,595 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:17:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-15T23:17:24,597 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:17:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T23:17:24,597 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:17:24,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:17:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T23:17:24,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T23:17:24,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:24,749 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T23:17:24,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:24,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:24,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/80d9ce7b7b864b10b7ac39662d39ebd7 is 50, key is test_row_1/A:col10/1731712643567/Put/seqid=0 2024-11-15T23:17:24,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742323_1499 (size=9857) 2024-11-15T23:17:24,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712644840, max timestamp=1731712644870Num uniques 859; 2024-11-15T23:17:24,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712644840, max timestamp=1731712644870Num uniques 859; 2024-11-15T23:17:24,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712644840, max timestamp=1731712644870Num uniques 859; 2024-11-15T23:17:24,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:17:24,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:17:24,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:17:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T23:17:25,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712705018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:25,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:25,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712705135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:25,162 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/80d9ce7b7b864b10b7ac39662d39ebd7 2024-11-15T23:17:25,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/5ae4c32046fc4b6eb6e9961546403365 is 50, key is test_row_1/B:col10/1731712643567/Put/seqid=0 2024-11-15T23:17:25,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742324_1500 (size=9857) 2024-11-15T23:17:25,176 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/5ae4c32046fc4b6eb6e9961546403365 2024-11-15T23:17:25,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/8649399432e241a1b7d12aca56085b89 is 50, key is test_row_1/C:col10/1731712643567/Put/seqid=0 2024-11-15T23:17:25,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742325_1501 (size=9857) 2024-11-15T23:17:25,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T23:17:25,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:25,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712705345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:25,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:25,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:25,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:25,592 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/8649399432e241a1b7d12aca56085b89 2024-11-15T23:17:25,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/80d9ce7b7b864b10b7ac39662d39ebd7 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/80d9ce7b7b864b10b7ac39662d39ebd7 2024-11-15T23:17:25,598 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/80d9ce7b7b864b10b7ac39662d39ebd7, entries=100, sequenceid=447, filesize=9.6 K 2024-11-15T23:17:25,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/5ae4c32046fc4b6eb6e9961546403365 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/5ae4c32046fc4b6eb6e9961546403365 2024-11-15T23:17:25,601 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/5ae4c32046fc4b6eb6e9961546403365, entries=100, sequenceid=447, filesize=9.6 K 2024-11-15T23:17:25,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/8649399432e241a1b7d12aca56085b89 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/8649399432e241a1b7d12aca56085b89 2024-11-15T23:17:25,605 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/8649399432e241a1b7d12aca56085b89, entries=100, sequenceid=447, filesize=9.6 K 2024-11-15T23:17:25,606 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3ad510e3715c6faa652057022b8e1f17 in 857ms, sequenceid=447, compaction requested=true 2024-11-15T23:17:25,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:25,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:25,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-15T23:17:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-15T23:17:25,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-15T23:17:25,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0100 sec 2024-11-15T23:17:25,610 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.0140 sec 2024-11-15T23:17:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:25,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-15T23:17:25,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:25,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T23:17:25,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:25,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T23:17:25,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:25,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T23:17:25,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/92c0407ae2084f45aba1c82587403450 is 50, key is test_row_0/A:col10/1731712645006/Put/seqid=0 2024-11-15T23:17:25,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742326_1502 (size=14741) 2024-11-15T23:17:25,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712705727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T23:17:25,737 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:25,738 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:17:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-15T23:17:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T23:17:25,739 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:17:25,740 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:17:25,740 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:17:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T23:17:25,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712705845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:25,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T23:17:25,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:25,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:25,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:25,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:25,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:25,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:26,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T23:17:26,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:26,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:26,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:26,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:26,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T23:17:26,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712706055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:26,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:26,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:26,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:26,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/92c0407ae2084f45aba1c82587403450 2024-11-15T23:17:26,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/07d5208447a249d38d7aecb4268a084b is 50, key is test_row_0/B:col10/1731712645006/Put/seqid=0 2024-11-15T23:17:26,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742327_1503 (size=12301) 2024-11-15T23:17:26,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T23:17:26,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:26,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:26,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:26,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:26,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:26,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T23:17:26,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:26,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:26,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:26,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:26,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T23:17:26,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712706376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:26,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/07d5208447a249d38d7aecb4268a084b 2024-11-15T23:17:26,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/46d8722ecc0c468d8270bab5cfb72e01 is 50, key is test_row_0/C:col10/1731712645006/Put/seqid=0 2024-11-15T23:17:26,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742328_1504 (size=12301) 2024-11-15T23:17:26,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T23:17:26,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:26,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:26,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:26,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:26,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:26,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/46d8722ecc0c468d8270bab5cfb72e01 2024-11-15T23:17:26,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/92c0407ae2084f45aba1c82587403450 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/92c0407ae2084f45aba1c82587403450 2024-11-15T23:17:26,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/92c0407ae2084f45aba1c82587403450, entries=200, sequenceid=485, filesize=14.4 K 2024-11-15T23:17:26,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/07d5208447a249d38d7aecb4268a084b as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/07d5208447a249d38d7aecb4268a084b 2024-11-15T23:17:26,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/07d5208447a249d38d7aecb4268a084b, entries=150, sequenceid=485, filesize=12.0 K 2024-11-15T23:17:26,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/46d8722ecc0c468d8270bab5cfb72e01 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/46d8722ecc0c468d8270bab5cfb72e01 2024-11-15T23:17:26,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/46d8722ecc0c468d8270bab5cfb72e01, entries=150, sequenceid=485, filesize=12.0 K 2024-11-15T23:17:26,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 3ad510e3715c6faa652057022b8e1f17 in 848ms, sequenceid=485, compaction requested=true 2024-11-15T23:17:26,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:26,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:17:26,519 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:17:26,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:26,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:17:26,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:26,520 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:17:26,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:17:26,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:26,521 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47408 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:17:26,521 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49848 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:17:26,521 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/A is initiating minor compaction (all files) 2024-11-15T23:17:26,521 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/B is initiating minor compaction (all files) 2024-11-15T23:17:26,521 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/A in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:26,521 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/B in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:26,521 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/3b4d6e1301e54b199c021acf18b715b2, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/36078ede98c447678675b60727bd0f93, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/80d9ce7b7b864b10b7ac39662d39ebd7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/92c0407ae2084f45aba1c82587403450] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=48.7 K 2024-11-15T23:17:26,521 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/5f486f0664a945dc98001550a96adfbf, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/4a11976230e641a0a6dc69bc79c0fd30, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/5ae4c32046fc4b6eb6e9961546403365, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/07d5208447a249d38d7aecb4268a084b] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=46.3 K 2024-11-15T23:17:26,521 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f486f0664a945dc98001550a96adfbf, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731712642086 2024-11-15T23:17:26,521 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b4d6e1301e54b199c021acf18b715b2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731712642086 2024-11-15T23:17:26,522 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a11976230e641a0a6dc69bc79c0fd30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731712642352 2024-11-15T23:17:26,522 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36078ede98c447678675b60727bd0f93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731712642352 2024-11-15T23:17:26,522 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ae4c32046fc4b6eb6e9961546403365, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731712643567 2024-11-15T23:17:26,522 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80d9ce7b7b864b10b7ac39662d39ebd7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731712643567 2024-11-15T23:17:26,523 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 07d5208447a249d38d7aecb4268a084b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731712644983 2024-11-15T23:17:26,523 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92c0407ae2084f45aba1c82587403450, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731712644983 2024-11-15T23:17:26,539 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#A#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:26,539 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/a284240d7302483e93fae9534771cad3 is 50, key is test_row_0/A:col10/1731712645006/Put/seqid=0 2024-11-15T23:17:26,547 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#B#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:26,547 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/bb9f462da1dc4c838acd5d147216938d is 50, key is test_row_0/B:col10/1731712645006/Put/seqid=0 2024-11-15T23:17:26,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742329_1505 (size=13085) 2024-11-15T23:17:26,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742330_1506 (size=13085) 2024-11-15T23:17:26,573 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/bb9f462da1dc4c838acd5d147216938d as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/bb9f462da1dc4c838acd5d147216938d 2024-11-15T23:17:26,581 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/B of 3ad510e3715c6faa652057022b8e1f17 into bb9f462da1dc4c838acd5d147216938d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:26,581 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:26,581 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/B, priority=12, startTime=1731712646519; duration=0sec 2024-11-15T23:17:26,581 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:26,581 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:B 2024-11-15T23:17:26,581 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:17:26,582 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47408 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:17:26,582 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/C is initiating minor compaction (all files) 2024-11-15T23:17:26,582 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/C in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:26,583 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/e962a0847d184ec6b8983d96b06e2ef3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/97d3fdbbc7dd4ef89c119dd2d4395127, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/8649399432e241a1b7d12aca56085b89, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/46d8722ecc0c468d8270bab5cfb72e01] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=46.3 K 2024-11-15T23:17:26,583 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting e962a0847d184ec6b8983d96b06e2ef3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731712642086 2024-11-15T23:17:26,583 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 97d3fdbbc7dd4ef89c119dd2d4395127, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731712642352 2024-11-15T23:17:26,583 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 8649399432e241a1b7d12aca56085b89, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731712643567 2024-11-15T23:17:26,583 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 46d8722ecc0c468d8270bab5cfb72e01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731712644983 2024-11-15T23:17:26,595 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#C#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:26,595 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/ab5d376b6d6c4039a26c8236b35f70ed is 50, key is test_row_0/C:col10/1731712645006/Put/seqid=0 2024-11-15T23:17:26,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742331_1507 (size=13085) 2024-11-15T23:17:26,618 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/ab5d376b6d6c4039a26c8236b35f70ed as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/ab5d376b6d6c4039a26c8236b35f70ed 2024-11-15T23:17:26,629 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/C of 3ad510e3715c6faa652057022b8e1f17 into ab5d376b6d6c4039a26c8236b35f70ed(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:26,629 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:26,629 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/C, priority=12, startTime=1731712646520; duration=0sec 2024-11-15T23:17:26,629 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:26,629 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:C 2024-11-15T23:17:26,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T23:17:26,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:26,656 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-15T23:17:26,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:26,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:26,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:26,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:26,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:26,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:26,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/432f072c9b0a4eb4ab433d2c762972db is 50, key is test_row_0/A:col10/1731712645687/Put/seqid=0 2024-11-15T23:17:26,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742332_1508 (size=12301) 2024-11-15T23:17:26,664 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/432f072c9b0a4eb4ab433d2c762972db 2024-11-15T23:17:26,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/ff9c642defc04134a56c0c511c2cb178 is 50, key is test_row_0/B:col10/1731712645687/Put/seqid=0 2024-11-15T23:17:26,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742333_1509 (size=12301) 2024-11-15T23:17:26,684 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/ff9c642defc04134a56c0c511c2cb178 2024-11-15T23:17:26,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/415bf4a18d25474f9d541793abaa2264 is 50, key is test_row_0/C:col10/1731712645687/Put/seqid=0 2024-11-15T23:17:26,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742334_1510 (size=12301) 2024-11-15T23:17:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T23:17:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:26,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:26,958 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/a284240d7302483e93fae9534771cad3 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/a284240d7302483e93fae9534771cad3 2024-11-15T23:17:26,961 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/A of 3ad510e3715c6faa652057022b8e1f17 into a284240d7302483e93fae9534771cad3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:26,961 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:26,961 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/A, priority=12, startTime=1731712646519; duration=0sec 2024-11-15T23:17:26,961 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:26,961 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:A 2024-11-15T23:17:27,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712646972, max timestamp=1731712647000Num uniques 859; 2024-11-15T23:17:27,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:17:27,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712646972, max timestamp=1731712647000Num uniques 859; 2024-11-15T23:17:27,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:17:27,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731712646972, max timestamp=1731712647000Num uniques 859; 2024-11-15T23:17:27,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T23:17:27,102 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/415bf4a18d25474f9d541793abaa2264 2024-11-15T23:17:27,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/432f072c9b0a4eb4ab433d2c762972db as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/432f072c9b0a4eb4ab433d2c762972db 2024-11-15T23:17:27,111 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/432f072c9b0a4eb4ab433d2c762972db, entries=150, sequenceid=496, filesize=12.0 K 2024-11-15T23:17:27,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/ff9c642defc04134a56c0c511c2cb178 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/ff9c642defc04134a56c0c511c2cb178 2024-11-15T23:17:27,116 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/ff9c642defc04134a56c0c511c2cb178, entries=150, sequenceid=496, filesize=12.0 K 2024-11-15T23:17:27,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/415bf4a18d25474f9d541793abaa2264 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/415bf4a18d25474f9d541793abaa2264 2024-11-15T23:17:27,119 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/415bf4a18d25474f9d541793abaa2264, entries=150, sequenceid=496, filesize=12.0 K 2024-11-15T23:17:27,120 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=134.18 KB/137400 for 3ad510e3715c6faa652057022b8e1f17 in 464ms, sequenceid=496, compaction requested=false 2024-11-15T23:17:27,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:27,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:27,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-15T23:17:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-15T23:17:27,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-15T23:17:27,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3810 sec 2024-11-15T23:17:27,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.3850 sec 2024-11-15T23:17:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:27,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-15T23:17:27,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:27,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T23:17:27,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:27,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T23:17:27,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:27,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T23:17:27,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/673b045e802e48c88631c066bccb7a44 is 50, key is test_row_0/A:col10/1731712647092/Put/seqid=0 2024-11-15T23:17:27,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742335_1511 (size=14741) 2024-11-15T23:17:27,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712707181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:27,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712707295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:27,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:27,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:27,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:27,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712707506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:27,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/673b045e802e48c88631c066bccb7a44 2024-11-15T23:17:27,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/db26376c45a24eaa9a474f3f35a6eedf is 50, key is test_row_0/B:col10/1731712647092/Put/seqid=0 2024-11-15T23:17:27,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742336_1512 (size=12301) 2024-11-15T23:17:27,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40364 deadline: 1731712707755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:27,766 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in 3ad510e3715c6faa652057022b8e1f17 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:17:27,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:27,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:27,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:27,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:27,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40286 deadline: 1731712707785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:27,793 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_2' in 3ad510e3715c6faa652057022b8e1f17 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:17:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:27,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40278 deadline: 1731712707805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:27,817 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_1' in 3ad510e3715c6faa652057022b8e1f17 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8275 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:17:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:27,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712707816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:27,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:27,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:27,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:27,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731712707816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:27,821 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d6ecb9b7dda1,38353,1731712503763 for 'test_row_0' in 3ad510e3715c6faa652057022b8e1f17 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8278 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T23:17:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T23:17:27,885 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:27,886 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:17:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-15T23:17:27,887 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:17:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T23:17:27,888 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:17:27,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:17:27,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/db26376c45a24eaa9a474f3f35a6eedf 2024-11-15T23:17:27,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/4bfee6fc75f24f1e98584d0995a48511 is 50, key is test_row_0/C:col10/1731712647092/Put/seqid=0 2024-11-15T23:17:27,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742337_1513 (size=12301) 2024-11-15T23:17:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T23:17:28,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T23:17:28,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:28,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:28,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:28,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:28,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:28,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T23:17:28,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:28,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:28,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:28,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:28,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T23:17:28,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712708326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:28,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:28,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:28,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:28,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T23:17:28,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:28,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:28,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:28,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:28,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:28,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/4bfee6fc75f24f1e98584d0995a48511 2024-11-15T23:17:28,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/673b045e802e48c88631c066bccb7a44 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/673b045e802e48c88631c066bccb7a44 2024-11-15T23:17:28,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/673b045e802e48c88631c066bccb7a44, entries=200, sequenceid=535, filesize=14.4 K 2024-11-15T23:17:28,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/db26376c45a24eaa9a474f3f35a6eedf as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/db26376c45a24eaa9a474f3f35a6eedf 2024-11-15T23:17:28,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/db26376c45a24eaa9a474f3f35a6eedf, entries=150, sequenceid=535, filesize=12.0 K 2024-11-15T23:17:28,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/4bfee6fc75f24f1e98584d0995a48511 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4bfee6fc75f24f1e98584d0995a48511 2024-11-15T23:17:28,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4bfee6fc75f24f1e98584d0995a48511, entries=150, sequenceid=535, filesize=12.0 K 2024-11-15T23:17:28,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 3ad510e3715c6faa652057022b8e1f17 in 1285ms, sequenceid=535, compaction requested=true 2024-11-15T23:17:28,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:28,415 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:28,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:17:28,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:28,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:17:28,415 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:28,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:28,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:17:28,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:28,415 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:28,416 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/A is initiating minor compaction (all files) 2024-11-15T23:17:28,416 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/A in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:28,416 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:28,416 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/a284240d7302483e93fae9534771cad3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/432f072c9b0a4eb4ab433d2c762972db, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/673b045e802e48c88631c066bccb7a44] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=39.2 K 2024-11-15T23:17:28,416 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/B is initiating minor compaction (all files) 2024-11-15T23:17:28,416 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/B in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:28,416 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/bb9f462da1dc4c838acd5d147216938d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/ff9c642defc04134a56c0c511c2cb178, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/db26376c45a24eaa9a474f3f35a6eedf] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=36.8 K 2024-11-15T23:17:28,418 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting a284240d7302483e93fae9534771cad3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731712644983 2024-11-15T23:17:28,418 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting bb9f462da1dc4c838acd5d147216938d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731712644983 2024-11-15T23:17:28,418 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 432f072c9b0a4eb4ab433d2c762972db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731712645687 2024-11-15T23:17:28,418 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting ff9c642defc04134a56c0c511c2cb178, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731712645687 2024-11-15T23:17:28,418 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting db26376c45a24eaa9a474f3f35a6eedf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731712647092 2024-11-15T23:17:28,418 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 673b045e802e48c88631c066bccb7a44, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731712647092 2024-11-15T23:17:28,424 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#B#compaction#440 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:28,424 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/19584df8e93c4b9eb3d24744db28f95f is 50, key is test_row_0/B:col10/1731712647092/Put/seqid=0 2024-11-15T23:17:28,449 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#A#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:28,450 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/5ebd9cc94ee74c9d872e0caf2063cb99 is 50, key is test_row_0/A:col10/1731712647092/Put/seqid=0 2024-11-15T23:17:28,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742338_1514 (size=13187) 2024-11-15T23:17:28,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742339_1515 (size=13187) 2024-11-15T23:17:28,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T23:17:28,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:28,498 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-15T23:17:28,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:28,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:28,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:28,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:28,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:28,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:28,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/bb21a9aef1844ba9a046ad48c38dd4f6 is 50, key is test_row_0/A:col10/1731712647150/Put/seqid=0 2024-11-15T23:17:28,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742340_1516 (size=9857) 2024-11-15T23:17:28,513 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/bb21a9aef1844ba9a046ad48c38dd4f6 2024-11-15T23:17:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T23:17:28,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/0956e4725c394a4c89369594de917ef4 is 50, key is test_row_0/B:col10/1731712647150/Put/seqid=0 2024-11-15T23:17:28,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742341_1517 (size=9857) 2024-11-15T23:17:28,870 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/5ebd9cc94ee74c9d872e0caf2063cb99 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/5ebd9cc94ee74c9d872e0caf2063cb99 2024-11-15T23:17:28,870 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/19584df8e93c4b9eb3d24744db28f95f as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/19584df8e93c4b9eb3d24744db28f95f 2024-11-15T23:17:28,877 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/A of 3ad510e3715c6faa652057022b8e1f17 into 5ebd9cc94ee74c9d872e0caf2063cb99(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:28,877 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/B of 3ad510e3715c6faa652057022b8e1f17 into 19584df8e93c4b9eb3d24744db28f95f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:28,877 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:28,877 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:28,877 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/A, priority=13, startTime=1731712648415; duration=0sec 2024-11-15T23:17:28,877 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/B, priority=13, startTime=1731712648415; duration=0sec 2024-11-15T23:17:28,877 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:28,877 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:B 2024-11-15T23:17:28,877 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:28,878 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:28,878 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:A 2024-11-15T23:17:28,878 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:28,878 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/C is initiating minor compaction (all files) 2024-11-15T23:17:28,878 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/C in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:28,878 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/ab5d376b6d6c4039a26c8236b35f70ed, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/415bf4a18d25474f9d541793abaa2264, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4bfee6fc75f24f1e98584d0995a48511] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=36.8 K 2024-11-15T23:17:28,878 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting ab5d376b6d6c4039a26c8236b35f70ed, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731712644983 2024-11-15T23:17:28,879 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 415bf4a18d25474f9d541793abaa2264, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731712645687 2024-11-15T23:17:28,879 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bfee6fc75f24f1e98584d0995a48511, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731712647092 2024-11-15T23:17:28,885 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#C#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:28,886 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/a98711980967478980a8dfbfb3a1181a is 50, key is test_row_0/C:col10/1731712647092/Put/seqid=0 2024-11-15T23:17:28,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742342_1518 (size=13187) 2024-11-15T23:17:28,915 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/a98711980967478980a8dfbfb3a1181a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/a98711980967478980a8dfbfb3a1181a 2024-11-15T23:17:28,926 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/C of 3ad510e3715c6faa652057022b8e1f17 into a98711980967478980a8dfbfb3a1181a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:28,927 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:28,927 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/C, priority=13, startTime=1731712648415; duration=0sec 2024-11-15T23:17:28,927 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:28,927 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:C 2024-11-15T23:17:28,951 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/0956e4725c394a4c89369594de917ef4 2024-11-15T23:17:28,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/f2490b45b03e4c119e6dd6fcd9db2c21 is 50, key is test_row_0/C:col10/1731712647150/Put/seqid=0 2024-11-15T23:17:29,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742343_1519 (size=9857) 2024-11-15T23:17:29,006 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/f2490b45b03e4c119e6dd6fcd9db2c21 2024-11-15T23:17:29,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/bb21a9aef1844ba9a046ad48c38dd4f6 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/bb21a9aef1844ba9a046ad48c38dd4f6 2024-11-15T23:17:29,019 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/bb21a9aef1844ba9a046ad48c38dd4f6, entries=100, sequenceid=544, filesize=9.6 K 2024-11-15T23:17:29,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/0956e4725c394a4c89369594de917ef4 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/0956e4725c394a4c89369594de917ef4 2024-11-15T23:17:29,024 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/0956e4725c394a4c89369594de917ef4, entries=100, sequenceid=544, filesize=9.6 K 2024-11-15T23:17:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T23:17:29,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/f2490b45b03e4c119e6dd6fcd9db2c21 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/f2490b45b03e4c119e6dd6fcd9db2c21 2024-11-15T23:17:29,038 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/f2490b45b03e4c119e6dd6fcd9db2c21, entries=100, sequenceid=544, filesize=9.6 K 2024-11-15T23:17:29,038 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 3ad510e3715c6faa652057022b8e1f17 in 540ms, sequenceid=544, compaction requested=false 2024-11-15T23:17:29,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:29,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:29,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-15T23:17:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-15T23:17:29,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-15T23:17:29,042 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1520 sec 2024-11-15T23:17:29,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.1570 sec 2024-11-15T23:17:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:29,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T23:17:29,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:29,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:29,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:29,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:29,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:29,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:29,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/94a140e37780485c869bce5a121e3191 is 50, key is test_row_1/A:col10/1731712649347/Put/seqid=0 2024-11-15T23:17:29,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742344_1520 (size=12297) 2024-11-15T23:17:29,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/94a140e37780485c869bce5a121e3191 2024-11-15T23:17:29,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/cc6b5cddd49b4fe78c24778aa6209964 is 50, key is test_row_1/B:col10/1731712649347/Put/seqid=0 2024-11-15T23:17:29,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742345_1521 (size=9857) 2024-11-15T23:17:29,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712649393, max timestamp=1731712649484Num uniques 859; 2024-11-15T23:17:29,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:17:29,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712649393, max timestamp=1731712649484Num uniques 859; 2024-11-15T23:17:29,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:17:29,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731712649393, max timestamp=1731712649484Num uniques 859; 2024-11-15T23:17:29,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6ecb9b7dda1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T23:17:29,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 332 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712709647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:29,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 333 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712709755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:29,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:29,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:29,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:29,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/cc6b5cddd49b4fe78c24778aa6209964 2024-11-15T23:17:29,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/0fdeec0e03544731ac42f118f1bb6e2d is 50, key is test_row_1/C:col10/1731712649347/Put/seqid=0 2024-11-15T23:17:29,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742346_1522 (size=9857) 2024-11-15T23:17:29,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 334 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712709965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:29,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:29,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:29,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T23:17:30,036 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:30,036 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:17:30,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-15T23:17:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T23:17:30,038 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:17:30,038 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:17:30,038 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:17:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T23:17:30,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T23:17:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:30,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:30,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:30,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:30,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/0fdeec0e03544731ac42f118f1bb6e2d 2024-11-15T23:17:30,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/94a140e37780485c869bce5a121e3191 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/94a140e37780485c869bce5a121e3191 2024-11-15T23:17:30,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/94a140e37780485c869bce5a121e3191, entries=150, sequenceid=557, filesize=12.0 K 2024-11-15T23:17:30,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/cc6b5cddd49b4fe78c24778aa6209964 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/cc6b5cddd49b4fe78c24778aa6209964 2024-11-15T23:17:30,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/cc6b5cddd49b4fe78c24778aa6209964, entries=100, sequenceid=557, filesize=9.6 K 2024-11-15T23:17:30,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/0fdeec0e03544731ac42f118f1bb6e2d as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/0fdeec0e03544731ac42f118f1bb6e2d 2024-11-15T23:17:30,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/0fdeec0e03544731ac42f118f1bb6e2d, entries=100, sequenceid=557, filesize=9.6 K 2024-11-15T23:17:30,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3ad510e3715c6faa652057022b8e1f17 in 847ms, sequenceid=557, compaction requested=true 2024-11-15T23:17:30,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:30,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:17:30,207 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:30,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:30,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:17:30,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:30,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ad510e3715c6faa652057022b8e1f17:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:17:30,207 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:30,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:30,208 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35341 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:30,208 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/A is initiating minor compaction (all files) 2024-11-15T23:17:30,208 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32901 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:30,208 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/A in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:30,208 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/B is initiating minor compaction (all files) 2024-11-15T23:17:30,208 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/B in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:30,208 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/5ebd9cc94ee74c9d872e0caf2063cb99, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/bb21a9aef1844ba9a046ad48c38dd4f6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/94a140e37780485c869bce5a121e3191] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=34.5 K 2024-11-15T23:17:30,208 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/19584df8e93c4b9eb3d24744db28f95f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/0956e4725c394a4c89369594de917ef4, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/cc6b5cddd49b4fe78c24778aa6209964] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=32.1 K 2024-11-15T23:17:30,209 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ebd9cc94ee74c9d872e0caf2063cb99, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731712647092 2024-11-15T23:17:30,209 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 19584df8e93c4b9eb3d24744db28f95f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731712647092 2024-11-15T23:17:30,209 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb21a9aef1844ba9a046ad48c38dd4f6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1731712647150 2024-11-15T23:17:30,209 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 0956e4725c394a4c89369594de917ef4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1731712647150 2024-11-15T23:17:30,209 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94a140e37780485c869bce5a121e3191, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=557, earliestPutTs=1731712649347 2024-11-15T23:17:30,209 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting cc6b5cddd49b4fe78c24778aa6209964, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=557, earliestPutTs=1731712649347 2024-11-15T23:17:30,215 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#A#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:30,215 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/d0f76d9a51904642ba9ee3df3e173eaa is 50, key is test_row_0/A:col10/1731712647150/Put/seqid=0 2024-11-15T23:17:30,224 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#B#compaction#450 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:30,225 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/3968ab4518d84016a3e247853d7ca317 is 50, key is test_row_0/B:col10/1731712647150/Put/seqid=0 2024-11-15T23:17:30,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742347_1523 (size=13289) 2024-11-15T23:17:30,237 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/d0f76d9a51904642ba9ee3df3e173eaa as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/d0f76d9a51904642ba9ee3df3e173eaa 2024-11-15T23:17:30,240 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/A of 3ad510e3715c6faa652057022b8e1f17 into d0f76d9a51904642ba9ee3df3e173eaa(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:30,240 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:30,240 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/A, priority=13, startTime=1731712650207; duration=0sec 2024-11-15T23:17:30,240 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:30,240 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:A 2024-11-15T23:17:30,240 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:30,241 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32901 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:30,241 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 3ad510e3715c6faa652057022b8e1f17/C is initiating minor compaction (all files) 2024-11-15T23:17:30,241 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ad510e3715c6faa652057022b8e1f17/C in TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:30,241 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/a98711980967478980a8dfbfb3a1181a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/f2490b45b03e4c119e6dd6fcd9db2c21, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/0fdeec0e03544731ac42f118f1bb6e2d] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp, totalSize=32.1 K 2024-11-15T23:17:30,241 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting a98711980967478980a8dfbfb3a1181a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731712647092 2024-11-15T23:17:30,241 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2490b45b03e4c119e6dd6fcd9db2c21, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1731712647150 2024-11-15T23:17:30,242 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fdeec0e03544731ac42f118f1bb6e2d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=557, earliestPutTs=1731712649347 2024-11-15T23:17:30,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742348_1524 (size=13289) 2024-11-15T23:17:30,249 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ad510e3715c6faa652057022b8e1f17#C#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:30,250 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/5145389678b248c88133dc509982ecb6 is 50, key is test_row_0/C:col10/1731712647150/Put/seqid=0 2024-11-15T23:17:30,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742349_1525 (size=13289) 2024-11-15T23:17:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:30,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-15T23:17:30,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:30,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:30,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:30,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:30,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:30,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T23:17:30,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/f9a2b6a13c3f48928986a3494bf06641 is 50, key is test_row_0/A:col10/1731712650280/Put/seqid=0 2024-11-15T23:17:30,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742350_1526 (size=14741) 2024-11-15T23:17:30,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=595 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/f9a2b6a13c3f48928986a3494bf06641 2024-11-15T23:17:30,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/0f328476ea4e43458cd6f062f6c36348 is 50, key is test_row_0/B:col10/1731712650280/Put/seqid=0 2024-11-15T23:17:30,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742351_1527 (size=12301) 2024-11-15T23:17:30,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=595 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/0f328476ea4e43458cd6f062f6c36348 2024-11-15T23:17:30,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/3dfa9bc5df4c4279948b6f243893d283 is 50, key is test_row_0/C:col10/1731712650280/Put/seqid=0 2024-11-15T23:17:30,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742352_1528 (size=12301) 2024-11-15T23:17:30,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T23:17:30,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:30,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:30,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:30,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:30,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:30,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 342 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712710332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:30,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:30,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:30,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T23:17:30,430 INFO [Thread-1973 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1973. 2024-11-15T23:17:30,430 DEBUG [Thread-1973 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:30,430 DEBUG [Thread-1973 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:30,430 DEBUG [Thread-1973 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:30,430 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:30,431 INFO [Thread-1975 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1975. 2024-11-15T23:17:30,431 DEBUG [Thread-1975 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:30,431 DEBUG [Thread-1975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:30,431 DEBUG [Thread-1975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:30,431 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:30,436 INFO [Thread-1977 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1977. 2024-11-15T23:17:30,436 DEBUG [Thread-1977 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:30,436 DEBUG [Thread-1977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:30,436 DEBUG [Thread-1977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:30,436 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:30,443 INFO [Thread-1971 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1971. 2024-11-15T23:17:30,443 DEBUG [Thread-1971 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:30,443 DEBUG [Thread-1971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:30,443 DEBUG [Thread-1971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:30,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:30,444 INFO [Thread-1979 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1979. 2024-11-15T23:17:30,445 DEBUG [Thread-1979 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:30,445 DEBUG [Thread-1979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:30,445 DEBUG [Thread-1979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:30,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:30,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:30,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 343 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712710455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:30,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T23:17:30,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:30,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:30,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:30,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:30,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:30,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T23:17:30,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:30,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:30,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:30,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:30,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:30,657 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/3968ab4518d84016a3e247853d7ca317 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/3968ab4518d84016a3e247853d7ca317 2024-11-15T23:17:30,662 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/5145389678b248c88133dc509982ecb6 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/5145389678b248c88133dc509982ecb6 2024-11-15T23:17:30,663 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/B of 3ad510e3715c6faa652057022b8e1f17 into 3968ab4518d84016a3e247853d7ca317(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:30,663 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:30,663 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/B, priority=13, startTime=1731712650207; duration=0sec 2024-11-15T23:17:30,663 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:30,663 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:B 2024-11-15T23:17:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T23:17:30,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 344 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40304 deadline: 1731712710665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 , the old value is region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:17:30,666 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ad510e3715c6faa652057022b8e1f17/C of 3ad510e3715c6faa652057022b8e1f17 into 5145389678b248c88133dc509982ecb6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:30,666 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:30,666 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., storeName=3ad510e3715c6faa652057022b8e1f17/C, priority=13, startTime=1731712650207; duration=0sec 2024-11-15T23:17:30,666 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:30,666 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ad510e3715c6faa652057022b8e1f17:C 2024-11-15T23:17:30,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=595 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/3dfa9bc5df4c4279948b6f243893d283 2024-11-15T23:17:30,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/f9a2b6a13c3f48928986a3494bf06641 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/f9a2b6a13c3f48928986a3494bf06641 2024-11-15T23:17:30,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/f9a2b6a13c3f48928986a3494bf06641, entries=200, sequenceid=595, filesize=14.4 K 2024-11-15T23:17:30,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/0f328476ea4e43458cd6f062f6c36348 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/0f328476ea4e43458cd6f062f6c36348 2024-11-15T23:17:30,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/0f328476ea4e43458cd6f062f6c36348, entries=150, sequenceid=595, filesize=12.0 K 2024-11-15T23:17:30,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/3dfa9bc5df4c4279948b6f243893d283 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/3dfa9bc5df4c4279948b6f243893d283 2024-11-15T23:17:30,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/3dfa9bc5df4c4279948b6f243893d283, entries=150, sequenceid=595, filesize=12.0 K 2024-11-15T23:17:30,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 3ad510e3715c6faa652057022b8e1f17 in 440ms, sequenceid=595, compaction requested=false 2024-11-15T23:17:30,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:30,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T23:17:30,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:30,803 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-15T23:17:30,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:30,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:30,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:30,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:30,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:30,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:30,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/8c1143583f024fb6a6e74a66209a2b05 is 50, key is test_row_0/A:col10/1731712650329/Put/seqid=0 2024-11-15T23:17:30,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742353_1529 (size=9857) 2024-11-15T23:17:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:30,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. as already flushing 2024-11-15T23:17:30,977 INFO [Thread-1962 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1962. 2024-11-15T23:17:30,978 DEBUG [Thread-1962 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:30,978 DEBUG [Thread-1962 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:30,978 DEBUG [Thread-1962 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:30,978 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T23:17:31,211 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/8c1143583f024fb6a6e74a66209a2b05 2024-11-15T23:17:31,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/b39847d4ce484eb685743b793881736c is 50, key is test_row_0/B:col10/1731712650329/Put/seqid=0 2024-11-15T23:17:31,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742354_1530 (size=9857) 2024-11-15T23:17:31,623 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/b39847d4ce484eb685743b793881736c 2024-11-15T23:17:31,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/3144ae4d8a504ed1829af1b888f59d96 is 50, key is test_row_0/C:col10/1731712650329/Put/seqid=0 2024-11-15T23:17:31,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742355_1531 (size=9857) 2024-11-15T23:17:31,968 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T23:17:32,041 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/3144ae4d8a504ed1829af1b888f59d96 2024-11-15T23:17:32,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/8c1143583f024fb6a6e74a66209a2b05 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/8c1143583f024fb6a6e74a66209a2b05 2024-11-15T23:17:32,054 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/8c1143583f024fb6a6e74a66209a2b05, entries=100, sequenceid=606, filesize=9.6 K 2024-11-15T23:17:32,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/b39847d4ce484eb685743b793881736c as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/b39847d4ce484eb685743b793881736c 2024-11-15T23:17:32,059 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/b39847d4ce484eb685743b793881736c, entries=100, sequenceid=606, filesize=9.6 K 2024-11-15T23:17:32,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/3144ae4d8a504ed1829af1b888f59d96 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/3144ae4d8a504ed1829af1b888f59d96 2024-11-15T23:17:32,064 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/3144ae4d8a504ed1829af1b888f59d96, entries=100, sequenceid=606, filesize=9.6 K 2024-11-15T23:17:32,065 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=6.71 KB/6870 for 3ad510e3715c6faa652057022b8e1f17 in 1262ms, sequenceid=606, compaction requested=true 2024-11-15T23:17:32,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 3ad510e3715c6faa652057022b8e1f17: 2024-11-15T23:17:32,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:32,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-15T23:17:32,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-15T23:17:32,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-15T23:17:32,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0280 sec 2024-11-15T23:17:32,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 2.0330 sec 2024-11-15T23:17:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T23:17:32,186 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:37,799 INFO [Thread-1968 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1968. 2024-11-15T23:17:37,799 DEBUG [Thread-1968 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:37,799 DEBUG [Thread-1968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:37,799 DEBUG [Thread-1968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:37,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:37,829 INFO [Thread-1964 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1964. 2024-11-15T23:17:37,829 DEBUG [Thread-1964 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:37,829 DEBUG [Thread-1964 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:37,829 DEBUG [Thread-1964 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:37,829 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:37,856 INFO [Thread-1960 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1960. 2024-11-15T23:17:37,856 DEBUG [Thread-1960 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:37,857 DEBUG [Thread-1960 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:37,857 DEBUG [Thread-1960 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:37,857 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:37,917 INFO [Thread-1966 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1966. 2024-11-15T23:17:37,917 DEBUG [Thread-1966 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:17:37,917 DEBUG [Thread-1966 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:37,917 DEBUG [Thread-1966 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:37,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:37,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T23:17:37,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-15T23:17:37,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 282 2024-11-15T23:17:37,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-15T23:17:37,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-15T23:17:37,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-15T23:17:37,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T23:17:37,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T23:17:37,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1184 2024-11-15T23:17:37,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3551 rows 2024-11-15T23:17:37,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1178 2024-11-15T23:17:37,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3534 rows 2024-11-15T23:17:37,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1188 2024-11-15T23:17:37,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3561 rows 2024-11-15T23:17:37,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1176 2024-11-15T23:17:37,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3527 rows 2024-11-15T23:17:37,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1179 2024-11-15T23:17:37,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3537 rows 2024-11-15T23:17:37,918 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T23:17:37,918 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T23:17:37,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:37,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:37,918 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:17:37,919 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T23:17:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T23:17:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T23:17:37,922 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712657922"}]},"ts":"1731712657922"} 2024-11-15T23:17:37,923 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T23:17:37,923 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T23:17:37,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T23:17:37,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ad510e3715c6faa652057022b8e1f17, UNASSIGN}] 2024-11-15T23:17:37,926 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ad510e3715c6faa652057022b8e1f17, UNASSIGN 2024-11-15T23:17:37,926 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=3ad510e3715c6faa652057022b8e1f17, regionState=CLOSING, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:37,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ad510e3715c6faa652057022b8e1f17, UNASSIGN because future has completed 2024-11-15T23:17:37,928 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T23:17:37,928 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763}] 2024-11-15T23:17:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T23:17:38,081 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:38,081 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T23:17:38,081 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 3ad510e3715c6faa652057022b8e1f17, disabling compactions & flushes 2024-11-15T23:17:38,081 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:38,081 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:38,081 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. after waiting 0 ms 2024-11-15T23:17:38,081 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:38,081 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing 3ad510e3715c6faa652057022b8e1f17 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-15T23:17:38,082 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=A 2024-11-15T23:17:38,082 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:38,082 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=B 2024-11-15T23:17:38,082 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:38,082 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ad510e3715c6faa652057022b8e1f17, store=C 2024-11-15T23:17:38,082 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:38,088 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/ff69a0d5cf794b399b08cb0b9c2b9b44 is 50, key is test_row_0/A:col10/1731712657915/Put/seqid=0 2024-11-15T23:17:38,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742356_1532 (size=12301) 2024-11-15T23:17:38,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T23:17:38,495 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/ff69a0d5cf794b399b08cb0b9c2b9b44 2024-11-15T23:17:38,500 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/d3995d7029014d709d48bb0f374c35f9 is 50, key is test_row_0/B:col10/1731712657915/Put/seqid=0 2024-11-15T23:17:38,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742357_1533 (size=12301) 2024-11-15T23:17:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T23:17:38,904 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/d3995d7029014d709d48bb0f374c35f9 2024-11-15T23:17:38,913 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/b9f3773a0986423bb561aa8ff55845bb is 50, key is test_row_0/C:col10/1731712657915/Put/seqid=0 2024-11-15T23:17:38,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742358_1534 (size=12301) 2024-11-15T23:17:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T23:17:39,317 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/b9f3773a0986423bb561aa8ff55845bb 2024-11-15T23:17:39,321 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/A/ff69a0d5cf794b399b08cb0b9c2b9b44 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/ff69a0d5cf794b399b08cb0b9c2b9b44 2024-11-15T23:17:39,325 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/ff69a0d5cf794b399b08cb0b9c2b9b44, entries=150, sequenceid=614, filesize=12.0 K 2024-11-15T23:17:39,326 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/B/d3995d7029014d709d48bb0f374c35f9 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/d3995d7029014d709d48bb0f374c35f9 2024-11-15T23:17:39,330 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/d3995d7029014d709d48bb0f374c35f9, entries=150, sequenceid=614, filesize=12.0 K 2024-11-15T23:17:39,332 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/.tmp/C/b9f3773a0986423bb561aa8ff55845bb as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/b9f3773a0986423bb561aa8ff55845bb 2024-11-15T23:17:39,336 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/b9f3773a0986423bb561aa8ff55845bb, entries=150, sequenceid=614, filesize=12.0 K 2024-11-15T23:17:39,337 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 3ad510e3715c6faa652057022b8e1f17 in 1256ms, sequenceid=614, compaction requested=true 2024-11-15T23:17:39,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/113b8928f8fe47438033e9cdc8e2f0f3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/c6bd8df6929549e7a0547d2abb4f4a9b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/ba7b94c3bd5d4afeb7bdaddbd63e2a64, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/2b645254f0c8499a8dd705aadb491535, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/7835944832a8495bb8a4de2f380a2237, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/0361f1f97afd4252bdc54ca44d6c4a0c, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/06ca1cfb449a4a6a875658e0ed034b09, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/f5aee2c6cab148f6afe6cdc782e62bd1, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/bdabe2ae55c54b8fa027864ae940470c, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/aaaa39d856524a68bdf67832bd6b6d82, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/b1fbc4d733a24a979f1754596a46db75, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/1844a16f41734560b3f0910ca0cc7ead, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/6083af8dcaf44b7b808559785f89029c, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/631352e365e14997a680804ff16cccd7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/61677d84073c4c97bdb0f3391e1c6837, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/47d2092ccea444e482e577204e1c8ed9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/5312aa893b2e47ed9a50da3c15e8d31a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/f6b1be0ff4d44482b3a66a3d08b1e598, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/b76d1701b20746be9ff20f06c0b81617, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/3b4d6e1301e54b199c021acf18b715b2, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/36078ede98c447678675b60727bd0f93, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/80d9ce7b7b864b10b7ac39662d39ebd7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/92c0407ae2084f45aba1c82587403450, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/a284240d7302483e93fae9534771cad3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/432f072c9b0a4eb4ab433d2c762972db, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/673b045e802e48c88631c066bccb7a44, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/5ebd9cc94ee74c9d872e0caf2063cb99, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/bb21a9aef1844ba9a046ad48c38dd4f6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/94a140e37780485c869bce5a121e3191] to archive 2024-11-15T23:17:39,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:17:39,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/113b8928f8fe47438033e9cdc8e2f0f3 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/113b8928f8fe47438033e9cdc8e2f0f3 2024-11-15T23:17:39,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/c6bd8df6929549e7a0547d2abb4f4a9b to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/c6bd8df6929549e7a0547d2abb4f4a9b 2024-11-15T23:17:39,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/ba7b94c3bd5d4afeb7bdaddbd63e2a64 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/ba7b94c3bd5d4afeb7bdaddbd63e2a64 2024-11-15T23:17:39,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/2b645254f0c8499a8dd705aadb491535 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/2b645254f0c8499a8dd705aadb491535 2024-11-15T23:17:39,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/7835944832a8495bb8a4de2f380a2237 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/7835944832a8495bb8a4de2f380a2237 2024-11-15T23:17:39,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/0361f1f97afd4252bdc54ca44d6c4a0c to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/0361f1f97afd4252bdc54ca44d6c4a0c 2024-11-15T23:17:39,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/06ca1cfb449a4a6a875658e0ed034b09 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/06ca1cfb449a4a6a875658e0ed034b09 2024-11-15T23:17:39,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/f5aee2c6cab148f6afe6cdc782e62bd1 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/f5aee2c6cab148f6afe6cdc782e62bd1 2024-11-15T23:17:39,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/bdabe2ae55c54b8fa027864ae940470c to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/bdabe2ae55c54b8fa027864ae940470c 2024-11-15T23:17:39,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/aaaa39d856524a68bdf67832bd6b6d82 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/aaaa39d856524a68bdf67832bd6b6d82 2024-11-15T23:17:39,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/b1fbc4d733a24a979f1754596a46db75 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/b1fbc4d733a24a979f1754596a46db75 2024-11-15T23:17:39,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/1844a16f41734560b3f0910ca0cc7ead to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/1844a16f41734560b3f0910ca0cc7ead 2024-11-15T23:17:39,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/6083af8dcaf44b7b808559785f89029c to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/6083af8dcaf44b7b808559785f89029c 2024-11-15T23:17:39,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/631352e365e14997a680804ff16cccd7 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/631352e365e14997a680804ff16cccd7 2024-11-15T23:17:39,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/61677d84073c4c97bdb0f3391e1c6837 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/61677d84073c4c97bdb0f3391e1c6837 2024-11-15T23:17:39,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/47d2092ccea444e482e577204e1c8ed9 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/47d2092ccea444e482e577204e1c8ed9 2024-11-15T23:17:39,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/5312aa893b2e47ed9a50da3c15e8d31a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/5312aa893b2e47ed9a50da3c15e8d31a 2024-11-15T23:17:39,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/f6b1be0ff4d44482b3a66a3d08b1e598 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/f6b1be0ff4d44482b3a66a3d08b1e598 2024-11-15T23:17:39,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/b76d1701b20746be9ff20f06c0b81617 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/b76d1701b20746be9ff20f06c0b81617 2024-11-15T23:17:39,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/3b4d6e1301e54b199c021acf18b715b2 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/3b4d6e1301e54b199c021acf18b715b2 2024-11-15T23:17:39,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/36078ede98c447678675b60727bd0f93 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/36078ede98c447678675b60727bd0f93 2024-11-15T23:17:39,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/80d9ce7b7b864b10b7ac39662d39ebd7 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/80d9ce7b7b864b10b7ac39662d39ebd7 2024-11-15T23:17:39,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/92c0407ae2084f45aba1c82587403450 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/92c0407ae2084f45aba1c82587403450 2024-11-15T23:17:39,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/a284240d7302483e93fae9534771cad3 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/a284240d7302483e93fae9534771cad3 2024-11-15T23:17:39,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/432f072c9b0a4eb4ab433d2c762972db to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/432f072c9b0a4eb4ab433d2c762972db 2024-11-15T23:17:39,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/673b045e802e48c88631c066bccb7a44 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/673b045e802e48c88631c066bccb7a44 2024-11-15T23:17:39,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/5ebd9cc94ee74c9d872e0caf2063cb99 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/5ebd9cc94ee74c9d872e0caf2063cb99 2024-11-15T23:17:39,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/bb21a9aef1844ba9a046ad48c38dd4f6 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/bb21a9aef1844ba9a046ad48c38dd4f6 2024-11-15T23:17:39,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/94a140e37780485c869bce5a121e3191 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/94a140e37780485c869bce5a121e3191 2024-11-15T23:17:39,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/4e29bbd020f24bc0975d170dc4749d04, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/ad83e68be27749968c65827d76f7ccd9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/2a513c0256e5410684b637b414f4b27d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/cad197f69ab344feb578c51ccf03e0c8, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/c3630c6cf49f4c92899207d1f897eff2, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/9139baef26064fda80175fb26658f333, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/9a88b40ccd0b4694979f9d60ffe47c81, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/640055d7dc27472b9f08becfb2b3b2bc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/3a6d4d071a614b05ae4305c7f724a0c6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/a82c0a476a8d46afb3b736e9d759f780, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/e1519e402e484ec78b57d96dac898044, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/c44686efd6ae4434bf4ede92a12d0aac, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/17b14d5f314f45939f5536b6a3b47b03, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/7374efeafa864390aa5500d6d151d939, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/edd2ad7fada245cbaf12daa3f193dca6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/95a93a867ca744a1b410b9cbaeeb09ff, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/ec616f00cc2642dbaceb81a961664867, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/91d42e86823b4339a1161ce121b26b09, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/5f486f0664a945dc98001550a96adfbf, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/8b3cec4203104e6fa3b7448c2f16e8bc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/4a11976230e641a0a6dc69bc79c0fd30, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/5ae4c32046fc4b6eb6e9961546403365, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/bb9f462da1dc4c838acd5d147216938d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/07d5208447a249d38d7aecb4268a084b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/ff9c642defc04134a56c0c511c2cb178, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/19584df8e93c4b9eb3d24744db28f95f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/db26376c45a24eaa9a474f3f35a6eedf, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/0956e4725c394a4c89369594de917ef4, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/cc6b5cddd49b4fe78c24778aa6209964] to archive 2024-11-15T23:17:39,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:17:39,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/4e29bbd020f24bc0975d170dc4749d04 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/4e29bbd020f24bc0975d170dc4749d04 2024-11-15T23:17:39,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/ad83e68be27749968c65827d76f7ccd9 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/ad83e68be27749968c65827d76f7ccd9 2024-11-15T23:17:39,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/2a513c0256e5410684b637b414f4b27d to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/2a513c0256e5410684b637b414f4b27d 2024-11-15T23:17:39,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/cad197f69ab344feb578c51ccf03e0c8 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/cad197f69ab344feb578c51ccf03e0c8 2024-11-15T23:17:39,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/c3630c6cf49f4c92899207d1f897eff2 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/c3630c6cf49f4c92899207d1f897eff2 2024-11-15T23:17:39,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/9139baef26064fda80175fb26658f333 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/9139baef26064fda80175fb26658f333 2024-11-15T23:17:39,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/9a88b40ccd0b4694979f9d60ffe47c81 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/9a88b40ccd0b4694979f9d60ffe47c81 2024-11-15T23:17:39,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/640055d7dc27472b9f08becfb2b3b2bc to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/640055d7dc27472b9f08becfb2b3b2bc 2024-11-15T23:17:39,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/3a6d4d071a614b05ae4305c7f724a0c6 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/3a6d4d071a614b05ae4305c7f724a0c6 2024-11-15T23:17:39,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/a82c0a476a8d46afb3b736e9d759f780 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/a82c0a476a8d46afb3b736e9d759f780 2024-11-15T23:17:39,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/e1519e402e484ec78b57d96dac898044 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/e1519e402e484ec78b57d96dac898044 2024-11-15T23:17:39,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/c44686efd6ae4434bf4ede92a12d0aac to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/c44686efd6ae4434bf4ede92a12d0aac 2024-11-15T23:17:39,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/17b14d5f314f45939f5536b6a3b47b03 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/17b14d5f314f45939f5536b6a3b47b03 2024-11-15T23:17:39,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/7374efeafa864390aa5500d6d151d939 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/7374efeafa864390aa5500d6d151d939 2024-11-15T23:17:39,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/edd2ad7fada245cbaf12daa3f193dca6 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/edd2ad7fada245cbaf12daa3f193dca6 2024-11-15T23:17:39,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/95a93a867ca744a1b410b9cbaeeb09ff to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/95a93a867ca744a1b410b9cbaeeb09ff 2024-11-15T23:17:39,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/ec616f00cc2642dbaceb81a961664867 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/ec616f00cc2642dbaceb81a961664867 2024-11-15T23:17:39,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/91d42e86823b4339a1161ce121b26b09 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/91d42e86823b4339a1161ce121b26b09 2024-11-15T23:17:39,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/5f486f0664a945dc98001550a96adfbf to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/5f486f0664a945dc98001550a96adfbf 2024-11-15T23:17:39,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/8b3cec4203104e6fa3b7448c2f16e8bc to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/8b3cec4203104e6fa3b7448c2f16e8bc 2024-11-15T23:17:39,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/4a11976230e641a0a6dc69bc79c0fd30 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/4a11976230e641a0a6dc69bc79c0fd30 2024-11-15T23:17:39,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/5ae4c32046fc4b6eb6e9961546403365 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/5ae4c32046fc4b6eb6e9961546403365 2024-11-15T23:17:39,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/bb9f462da1dc4c838acd5d147216938d to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/bb9f462da1dc4c838acd5d147216938d 2024-11-15T23:17:39,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/07d5208447a249d38d7aecb4268a084b to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/07d5208447a249d38d7aecb4268a084b 2024-11-15T23:17:39,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/ff9c642defc04134a56c0c511c2cb178 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/ff9c642defc04134a56c0c511c2cb178 2024-11-15T23:17:39,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/19584df8e93c4b9eb3d24744db28f95f to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/19584df8e93c4b9eb3d24744db28f95f 2024-11-15T23:17:39,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/db26376c45a24eaa9a474f3f35a6eedf to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/db26376c45a24eaa9a474f3f35a6eedf 2024-11-15T23:17:39,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/0956e4725c394a4c89369594de917ef4 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/0956e4725c394a4c89369594de917ef4 2024-11-15T23:17:39,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/cc6b5cddd49b4fe78c24778aa6209964 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/cc6b5cddd49b4fe78c24778aa6209964 2024-11-15T23:17:39,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/9d2227b1c3544551af1633c8ec33f640, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/e57ca7ca2fd842278e8f83f8e913a523, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/789ac68bdef54405b548db65e58dec5e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/ccc087c662bb47bf887ec3e97e8bb21a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4ef4cc7a28bd4c0390db2e1458462d54, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/33b0d42b6ce849ecbd43f7d71ff8757c, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/2eeff63e8d9e4a8dac69caeb6dcdd686, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/195f597632164ca5a6c8a0202eedc709, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/de43399a62834383bd579aeb92ecce9f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/7c1ac42eea6646a59e16a8bfe06f4c49, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/813e8f9223434b9fbfbc5fced5edd24e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/cfb89b421f34406aaf2fe2fba4d96772, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/b456fc1d0e18469fa1b4a26eb3bdea8b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4f252661893343c689f429f7a20f0c14, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/9ba42adc39fc4c738f0e7fdb6ff98f0b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/685ac11ecfb143cd9ec9aaa3891911a5, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/227c0203ab8241ca9ddbd4d8e7958f66, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/b29ca0bb3a024cd49619d7ea6f36a40e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/e962a0847d184ec6b8983d96b06e2ef3, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4aa4f045a9cb46ceace51ccab0fddab6, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/97d3fdbbc7dd4ef89c119dd2d4395127, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/8649399432e241a1b7d12aca56085b89, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/ab5d376b6d6c4039a26c8236b35f70ed, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/46d8722ecc0c468d8270bab5cfb72e01, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/415bf4a18d25474f9d541793abaa2264, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/a98711980967478980a8dfbfb3a1181a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4bfee6fc75f24f1e98584d0995a48511, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/f2490b45b03e4c119e6dd6fcd9db2c21, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/0fdeec0e03544731ac42f118f1bb6e2d] to archive 2024-11-15T23:17:39,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:17:39,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/9d2227b1c3544551af1633c8ec33f640 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/9d2227b1c3544551af1633c8ec33f640 2024-11-15T23:17:39,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/e57ca7ca2fd842278e8f83f8e913a523 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/e57ca7ca2fd842278e8f83f8e913a523 2024-11-15T23:17:39,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/789ac68bdef54405b548db65e58dec5e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/789ac68bdef54405b548db65e58dec5e 2024-11-15T23:17:39,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/ccc087c662bb47bf887ec3e97e8bb21a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/ccc087c662bb47bf887ec3e97e8bb21a 2024-11-15T23:17:39,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4ef4cc7a28bd4c0390db2e1458462d54 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4ef4cc7a28bd4c0390db2e1458462d54 2024-11-15T23:17:39,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/33b0d42b6ce849ecbd43f7d71ff8757c to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/33b0d42b6ce849ecbd43f7d71ff8757c 2024-11-15T23:17:39,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/2eeff63e8d9e4a8dac69caeb6dcdd686 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/2eeff63e8d9e4a8dac69caeb6dcdd686 2024-11-15T23:17:39,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/195f597632164ca5a6c8a0202eedc709 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/195f597632164ca5a6c8a0202eedc709 2024-11-15T23:17:39,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/de43399a62834383bd579aeb92ecce9f to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/de43399a62834383bd579aeb92ecce9f 2024-11-15T23:17:39,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/7c1ac42eea6646a59e16a8bfe06f4c49 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/7c1ac42eea6646a59e16a8bfe06f4c49 2024-11-15T23:17:39,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/813e8f9223434b9fbfbc5fced5edd24e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/813e8f9223434b9fbfbc5fced5edd24e 2024-11-15T23:17:39,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/cfb89b421f34406aaf2fe2fba4d96772 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/cfb89b421f34406aaf2fe2fba4d96772 2024-11-15T23:17:39,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/b456fc1d0e18469fa1b4a26eb3bdea8b to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/b456fc1d0e18469fa1b4a26eb3bdea8b 2024-11-15T23:17:39,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4f252661893343c689f429f7a20f0c14 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4f252661893343c689f429f7a20f0c14 2024-11-15T23:17:39,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/9ba42adc39fc4c738f0e7fdb6ff98f0b to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/9ba42adc39fc4c738f0e7fdb6ff98f0b 2024-11-15T23:17:39,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/685ac11ecfb143cd9ec9aaa3891911a5 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/685ac11ecfb143cd9ec9aaa3891911a5 2024-11-15T23:17:39,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/227c0203ab8241ca9ddbd4d8e7958f66 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/227c0203ab8241ca9ddbd4d8e7958f66 2024-11-15T23:17:39,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/b29ca0bb3a024cd49619d7ea6f36a40e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/b29ca0bb3a024cd49619d7ea6f36a40e 2024-11-15T23:17:39,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/e962a0847d184ec6b8983d96b06e2ef3 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/e962a0847d184ec6b8983d96b06e2ef3 2024-11-15T23:17:39,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4aa4f045a9cb46ceace51ccab0fddab6 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4aa4f045a9cb46ceace51ccab0fddab6 2024-11-15T23:17:39,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/97d3fdbbc7dd4ef89c119dd2d4395127 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/97d3fdbbc7dd4ef89c119dd2d4395127 2024-11-15T23:17:39,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/8649399432e241a1b7d12aca56085b89 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/8649399432e241a1b7d12aca56085b89 2024-11-15T23:17:39,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/ab5d376b6d6c4039a26c8236b35f70ed to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/ab5d376b6d6c4039a26c8236b35f70ed 2024-11-15T23:17:39,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/46d8722ecc0c468d8270bab5cfb72e01 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/46d8722ecc0c468d8270bab5cfb72e01 2024-11-15T23:17:39,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/415bf4a18d25474f9d541793abaa2264 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/415bf4a18d25474f9d541793abaa2264 2024-11-15T23:17:39,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/a98711980967478980a8dfbfb3a1181a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/a98711980967478980a8dfbfb3a1181a 2024-11-15T23:17:39,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4bfee6fc75f24f1e98584d0995a48511 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/4bfee6fc75f24f1e98584d0995a48511 2024-11-15T23:17:39,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/f2490b45b03e4c119e6dd6fcd9db2c21 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/f2490b45b03e4c119e6dd6fcd9db2c21 2024-11-15T23:17:39,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/0fdeec0e03544731ac42f118f1bb6e2d to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/0fdeec0e03544731ac42f118f1bb6e2d 2024-11-15T23:17:39,446 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/recovered.edits/617.seqid, newMaxSeqId=617, maxSeqId=1 2024-11-15T23:17:39,446 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. 2024-11-15T23:17:39,446 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 3ad510e3715c6faa652057022b8e1f17: Waiting for close lock at 1731712658081Running coprocessor pre-close hooks at 1731712658081Disabling compacts and flushes for region at 1731712658081Disabling writes for close at 1731712658081Obtaining lock to block concurrent updates at 1731712658081Preparing flush snapshotting stores in 3ad510e3715c6faa652057022b8e1f17 at 1731712658081Finished memstore snapshotting TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731712658082 (+1 ms)Flushing stores of TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17. at 1731712658083 (+1 ms)Flushing 3ad510e3715c6faa652057022b8e1f17/A: creating writer at 1731712658083Flushing 3ad510e3715c6faa652057022b8e1f17/A: appending metadata at 1731712658087 (+4 ms)Flushing 3ad510e3715c6faa652057022b8e1f17/A: closing flushed file at 1731712658087Flushing 3ad510e3715c6faa652057022b8e1f17/B: creating writer at 1731712658498 (+411 ms)Flushing 3ad510e3715c6faa652057022b8e1f17/B: appending metadata at 1731712658500 (+2 ms)Flushing 3ad510e3715c6faa652057022b8e1f17/B: closing flushed file at 1731712658500Flushing 3ad510e3715c6faa652057022b8e1f17/C: creating writer at 1731712658909 (+409 ms)Flushing 3ad510e3715c6faa652057022b8e1f17/C: appending metadata at 1731712658912 (+3 ms)Flushing 3ad510e3715c6faa652057022b8e1f17/C: closing flushed file at 1731712658912Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7de83746: reopening flushed file at 1731712659321 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46cf4b2: reopening flushed file at 1731712659325 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@770e2015: reopening flushed file at 1731712659330 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 3ad510e3715c6faa652057022b8e1f17 in 1256ms, sequenceid=614, compaction requested=true at 1731712659337 (+7 ms)Writing region close event to WAL at 1731712659443 (+106 ms)Running coprocessor post-close hooks at 1731712659446 (+3 ms)Closed at 1731712659446 2024-11-15T23:17:39,448 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:39,448 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=3ad510e3715c6faa652057022b8e1f17, regionState=CLOSED 2024-11-15T23:17:39,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 because future has completed 2024-11-15T23:17:39,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-15T23:17:39,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 3ad510e3715c6faa652057022b8e1f17, server=d6ecb9b7dda1,38353,1731712503763 in 1.5220 sec 2024-11-15T23:17:39,452 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-15T23:17:39,452 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ad510e3715c6faa652057022b8e1f17, UNASSIGN in 1.5260 sec 2024-11-15T23:17:39,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-15T23:17:39,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5280 sec 2024-11-15T23:17:39,454 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712659454"}]},"ts":"1731712659454"} 2024-11-15T23:17:39,455 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T23:17:39,455 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T23:17:39,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5370 sec 2024-11-15T23:17:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T23:17:40,066 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:40,066 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T23:17:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:17:40,068 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:17:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T23:17:40,068 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:17:40,070 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:40,072 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A, FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B, FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C, FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/recovered.edits] 2024-11-15T23:17:40,074 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/8c1143583f024fb6a6e74a66209a2b05 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/8c1143583f024fb6a6e74a66209a2b05 2024-11-15T23:17:40,075 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/d0f76d9a51904642ba9ee3df3e173eaa to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/d0f76d9a51904642ba9ee3df3e173eaa 2024-11-15T23:17:40,075 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/f9a2b6a13c3f48928986a3494bf06641 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/f9a2b6a13c3f48928986a3494bf06641 2024-11-15T23:17:40,076 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/ff69a0d5cf794b399b08cb0b9c2b9b44 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/A/ff69a0d5cf794b399b08cb0b9c2b9b44 2024-11-15T23:17:40,078 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/0f328476ea4e43458cd6f062f6c36348 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/0f328476ea4e43458cd6f062f6c36348 2024-11-15T23:17:40,079 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/3968ab4518d84016a3e247853d7ca317 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/3968ab4518d84016a3e247853d7ca317 2024-11-15T23:17:40,079 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/b39847d4ce484eb685743b793881736c to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/b39847d4ce484eb685743b793881736c 2024-11-15T23:17:40,080 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/d3995d7029014d709d48bb0f374c35f9 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/B/d3995d7029014d709d48bb0f374c35f9 2024-11-15T23:17:40,082 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/3144ae4d8a504ed1829af1b888f59d96 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/3144ae4d8a504ed1829af1b888f59d96 2024-11-15T23:17:40,082 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/3dfa9bc5df4c4279948b6f243893d283 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/3dfa9bc5df4c4279948b6f243893d283 2024-11-15T23:17:40,083 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/5145389678b248c88133dc509982ecb6 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/5145389678b248c88133dc509982ecb6 2024-11-15T23:17:40,084 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/b9f3773a0986423bb561aa8ff55845bb to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/C/b9f3773a0986423bb561aa8ff55845bb 2024-11-15T23:17:40,085 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/recovered.edits/617.seqid to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17/recovered.edits/617.seqid 2024-11-15T23:17:40,086 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/3ad510e3715c6faa652057022b8e1f17 2024-11-15T23:17:40,086 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T23:17:40,087 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:17:40,089 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T23:17:40,091 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T23:17:40,092 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:17:40,092 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T23:17:40,092 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731712660092"}]},"ts":"9223372036854775807"} 2024-11-15T23:17:40,094 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T23:17:40,094 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3ad510e3715c6faa652057022b8e1f17, NAME => 'TestAcidGuarantees,,1731712628211.3ad510e3715c6faa652057022b8e1f17.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T23:17:40,094 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T23:17:40,094 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731712660094"}]},"ts":"9223372036854775807"} 2024-11-15T23:17:40,095 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T23:17:40,096 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:17:40,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 30 msec 2024-11-15T23:17:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T23:17:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T23:17:40,178 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:40,186 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248 (was 248), OpenFileDescriptor=445 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=387 (was 421), ProcessCount=11 (was 11), AvailableMemoryMB=6090 (was 5525) - AvailableMemoryMB LEAK? - 2024-11-15T23:17:40,196 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=387, ProcessCount=11, AvailableMemoryMB=6089 2024-11-15T23:17:40,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T23:17:40,197 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T23:17:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T23:17:40,199 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T23:17:40,199 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:40,199 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-11-15T23:17:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T23:17:40,199 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T23:17:40,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742359_1535 (size=1061) 2024-11-15T23:17:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T23:17:40,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T23:17:40,607 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444 2024-11-15T23:17:40,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742360_1536 (size=53) 2024-11-15T23:17:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T23:17:41,014 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:17:41,014 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 06815b48d79c548c6fc12bf365c2f5fd, disabling compactions & flushes 2024-11-15T23:17:41,014 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:41,014 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:41,014 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. after waiting 0 ms 2024-11-15T23:17:41,014 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:41,014 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:41,015 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 06815b48d79c548c6fc12bf365c2f5fd: Waiting for close lock at 1731712661014Disabling compacts and flushes for region at 1731712661014Disabling writes for close at 1731712661014Writing region close event to WAL at 1731712661014Closed at 1731712661014 2024-11-15T23:17:41,015 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T23:17:41,016 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731712661016"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731712661016"}]},"ts":"1731712661016"} 2024-11-15T23:17:41,017 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T23:17:41,018 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T23:17:41,018 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712661018"}]},"ts":"1731712661018"} 2024-11-15T23:17:41,020 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T23:17:41,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=06815b48d79c548c6fc12bf365c2f5fd, ASSIGN}] 2024-11-15T23:17:41,021 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=06815b48d79c548c6fc12bf365c2f5fd, ASSIGN 2024-11-15T23:17:41,022 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=06815b48d79c548c6fc12bf365c2f5fd, ASSIGN; state=OFFLINE, location=d6ecb9b7dda1,38353,1731712503763; forceNewPlan=false, retain=false 2024-11-15T23:17:41,173 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=06815b48d79c548c6fc12bf365c2f5fd, regionState=OPENING, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:41,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=06815b48d79c548c6fc12bf365c2f5fd, ASSIGN because future has completed 2024-11-15T23:17:41,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763}] 2024-11-15T23:17:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T23:17:41,337 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:41,338 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} 2024-11-15T23:17:41,338 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:41,338 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:17:41,339 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:41,339 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:41,341 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:41,344 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:17:41,344 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06815b48d79c548c6fc12bf365c2f5fd columnFamilyName A 2024-11-15T23:17:41,344 DEBUG [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:41,345 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] regionserver.HStore(327): Store=06815b48d79c548c6fc12bf365c2f5fd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:17:41,345 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:41,347 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:17:41,348 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06815b48d79c548c6fc12bf365c2f5fd columnFamilyName B 2024-11-15T23:17:41,348 DEBUG [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:41,348 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] regionserver.HStore(327): Store=06815b48d79c548c6fc12bf365c2f5fd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:17:41,348 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:41,349 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:17:41,349 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06815b48d79c548c6fc12bf365c2f5fd columnFamilyName C 2024-11-15T23:17:41,349 DEBUG [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:41,350 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] regionserver.HStore(327): Store=06815b48d79c548c6fc12bf365c2f5fd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:17:41,350 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:41,350 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:41,351 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:41,351 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:41,352 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:41,352 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:41,352 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T23:17:41,353 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:41,355 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T23:17:41,355 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened 06815b48d79c548c6fc12bf365c2f5fd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70593417, jitterRate=0.05192388594150543}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T23:17:41,355 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:41,355 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for 06815b48d79c548c6fc12bf365c2f5fd: Running coprocessor pre-open hook at 1731712661339Writing region info on filesystem at 1731712661339Initializing all the Stores at 1731712661340 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712661340Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712661341 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712661341Cleaning up temporary data from old regions at 1731712661352 (+11 ms)Running coprocessor post-open hooks at 1731712661355 (+3 ms)Region opened successfully at 1731712661355 2024-11-15T23:17:41,356 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., pid=146, masterSystemTime=1731712661332 2024-11-15T23:17:41,357 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:41,358 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:41,358 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=06815b48d79c548c6fc12bf365c2f5fd, regionState=OPEN, openSeqNum=2, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:41,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 because future has completed 2024-11-15T23:17:41,361 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-15T23:17:41,361 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 in 183 msec 2024-11-15T23:17:41,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-15T23:17:41,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=06815b48d79c548c6fc12bf365c2f5fd, ASSIGN in 341 msec 2024-11-15T23:17:41,364 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T23:17:41,364 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712661364"}]},"ts":"1731712661364"} 2024-11-15T23:17:41,365 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T23:17:41,366 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T23:17:41,367 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-15T23:17:42,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T23:17:42,356 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:42,357 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2cba0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:42,357 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:42,357 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:42,358 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:42,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:42,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:42,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@404254bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:42,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:42,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:42,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:42,360 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:42,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18dd5f99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:42,361 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:42,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:42,363 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T23:17:42,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is d6ecb9b7dda1,43421,1731712502855 2024-11-15T23:17:42,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1f3c1b32 2024-11-15T23:17:42,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T23:17:42,365 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T23:17:42,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T23:17:42,367 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T23:17:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T23:17:42,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742361_1537 (size=1097) 2024-11-15T23:17:42,787 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-15T23:17:42,788 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-15T23:17:42,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T23:17:42,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=06815b48d79c548c6fc12bf365c2f5fd, REOPEN/MOVE}] 2024-11-15T23:17:42,796 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=06815b48d79c548c6fc12bf365c2f5fd, REOPEN/MOVE 2024-11-15T23:17:42,797 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=06815b48d79c548c6fc12bf365c2f5fd, regionState=CLOSING, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:42,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=06815b48d79c548c6fc12bf365c2f5fd, REOPEN/MOVE because future has completed 2024-11-15T23:17:42,798 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T23:17:42,799 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763}] 2024-11-15T23:17:42,951 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:42,951 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T23:17:42,951 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 06815b48d79c548c6fc12bf365c2f5fd, disabling compactions & flushes 2024-11-15T23:17:42,951 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:42,951 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:42,951 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. after waiting 0 ms 2024-11-15T23:17:42,951 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:42,956 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T23:17:42,957 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:42,957 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 06815b48d79c548c6fc12bf365c2f5fd: Waiting for close lock at 1731712662951Running coprocessor pre-close hooks at 1731712662951Disabling compacts and flushes for region at 1731712662951Disabling writes for close at 1731712662951Writing region close event to WAL at 1731712662952 (+1 ms)Running coprocessor post-close hooks at 1731712662957 (+5 ms)Closed at 1731712662957 2024-11-15T23:17:42,957 WARN [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: 06815b48d79c548c6fc12bf365c2f5fd to self. 2024-11-15T23:17:42,959 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:42,960 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=06815b48d79c548c6fc12bf365c2f5fd, regionState=CLOSED 2024-11-15T23:17:42,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 because future has completed 2024-11-15T23:17:42,966 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-15T23:17:42,966 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 in 164 msec 2024-11-15T23:17:42,967 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=06815b48d79c548c6fc12bf365c2f5fd, REOPEN/MOVE; state=CLOSED, location=d6ecb9b7dda1,38353,1731712503763; forceNewPlan=false, retain=true 2024-11-15T23:17:43,117 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=06815b48d79c548c6fc12bf365c2f5fd, regionState=OPENING, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:43,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=06815b48d79c548c6fc12bf365c2f5fd, REOPEN/MOVE because future has completed 2024-11-15T23:17:43,119 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763}] 2024-11-15T23:17:43,202 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T23:17:43,273 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:43,273 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} 2024-11-15T23:17:43,273 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:43,273 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:17:43,273 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:43,273 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:43,275 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:43,275 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:17:43,275 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06815b48d79c548c6fc12bf365c2f5fd columnFamilyName A 2024-11-15T23:17:43,276 DEBUG [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:43,277 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] regionserver.HStore(327): Store=06815b48d79c548c6fc12bf365c2f5fd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:17:43,277 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:43,277 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:17:43,277 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06815b48d79c548c6fc12bf365c2f5fd columnFamilyName B 2024-11-15T23:17:43,277 DEBUG [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:43,278 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] regionserver.HStore(327): Store=06815b48d79c548c6fc12bf365c2f5fd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:17:43,278 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:43,278 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T23:17:43,278 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06815b48d79c548c6fc12bf365c2f5fd columnFamilyName C 2024-11-15T23:17:43,279 DEBUG [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:43,279 INFO [StoreOpener-06815b48d79c548c6fc12bf365c2f5fd-1 {}] regionserver.HStore(327): Store=06815b48d79c548c6fc12bf365c2f5fd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:17:43,279 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:43,279 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:43,280 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:43,280 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:43,281 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:43,281 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:43,282 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T23:17:43,283 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:43,283 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened 06815b48d79c548c6fc12bf365c2f5fd; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73464814, jitterRate=0.09471103549003601}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T23:17:43,283 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:43,284 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for 06815b48d79c548c6fc12bf365c2f5fd: Running coprocessor pre-open hook at 1731712663273Writing region info on filesystem at 1731712663273Initializing all the Stores at 1731712663274 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712663274Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712663274Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731712663274Cleaning up temporary data from old regions at 1731712663281 (+7 ms)Running coprocessor post-open hooks at 1731712663283 (+2 ms)Region opened successfully at 1731712663284 (+1 ms) 2024-11-15T23:17:43,285 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., pid=151, masterSystemTime=1731712663270 2024-11-15T23:17:43,286 DEBUG [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:43,286 INFO [RS_OPEN_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:43,287 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=06815b48d79c548c6fc12bf365c2f5fd, regionState=OPEN, openSeqNum=5, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:43,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 because future has completed 2024-11-15T23:17:43,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-11-15T23:17:43,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 in 170 msec 2024-11-15T23:17:43,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-15T23:17:43,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=06815b48d79c548c6fc12bf365c2f5fd, REOPEN/MOVE in 496 msec 2024-11-15T23:17:43,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-15T23:17:43,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 503 msec 2024-11-15T23:17:43,300 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 931 msec 2024-11-15T23:17:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-15T23:17:43,301 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:43,302 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,302 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:43,302 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:43,303 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:43,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:43,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:43,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a774e18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:43,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:43,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:43,305 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:43,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41abae3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,306 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:43,306 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:43,307 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:43,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:43,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dd3f4e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:43,309 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:43,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e8ebf68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,311 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:43,311 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:43,312 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:43,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:43,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:43,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@210418dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:43,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:43,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:43,314 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:43,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c63d5e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,315 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,315 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:43,315 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:43,316 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53d87532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:43,317 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e136a38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,319 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:43,319 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:43,319 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d8b499d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:43,321 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:43,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@491fbe28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,322 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:43,322 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:43,323 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:43,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:43,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:43,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76adb8ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:43,324 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:43,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ed6a549, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,326 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:43,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:43,326 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f8c8d25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:43,328 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:43,329 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:43,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a7a66c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,330 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:43,331 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:43,331 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:43,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:43,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:43,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff43a8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:43,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:43,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:43,333 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@599327bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,335 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:43,335 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:43,335 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:43,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:43,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:43,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ef48d6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:43,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:43,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:43,337 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:43,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1be7f584, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,338 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6ecb9b7dda1,43421,-1 for getting cluster id 2024-11-15T23:17:43,338 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:17:43,339 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddc94e31-6920-474d-af61-ead96ac60460' 2024-11-15T23:17:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:17:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddc94e31-6920-474d-af61-ead96ac60460" 2024-11-15T23:17:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df561e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6ecb9b7dda1,43421,-1] 2024-11-15T23:17:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:17:43,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:17:43,340 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:17:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a746ea6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:17:43,341 DEBUG [Thread-2412 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:43,341 DEBUG [Thread-2397 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:43,341 DEBUG [Thread-2414 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:43,341 DEBUG [Thread-2401 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:43,341 DEBUG [Thread-2405 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:43,342 DEBUG [Thread-2416 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:43,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:43,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:43,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:43,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:43,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:43,344 DEBUG [Thread-2408 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:43,344 DEBUG [Thread-2403 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:43,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:43,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:43,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:43,344 DEBUG [Thread-2410 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:43,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:43,344 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:17:43,344 DEBUG [Thread-2399 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:17:43,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:43,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:43,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:43,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:43,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-15T23:17:43,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:17:43,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T23:17:43,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:43,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:43,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:17:43,346 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:17:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=-1] 2024-11-15T23:17:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:17:43,347 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:17:43,347 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:43,347 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:17:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:17:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:17:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:17:43,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:17:43,348 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:43,348 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:17:43,351 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:17:43,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:17:43,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:17:43,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5] 2024-11-15T23:17:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:43,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06815b48d79c548c6fc12bf365c2f5fd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T23:17:43,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=A 2024-11-15T23:17:43,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:43,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=B 2024-11-15T23:17:43,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:43,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=C 2024-11-15T23:17:43,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:43,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158e8a830540424cdabc553a35083a6347_06815b48d79c548c6fc12bf365c2f5fd is 50, key is test_row_0/A:col10/1731712663356/Put/seqid=0 2024-11-15T23:17:43,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712723387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:43,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:43,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712723389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712723388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:43,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712723392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712723391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:43,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742362_1538 (size=14594) 2024-11-15T23:17:43,402 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:43,404 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158e8a830540424cdabc553a35083a6347_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158e8a830540424cdabc553a35083a6347_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:43,405 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/863dcdda3ffe40198d91c3f7a06fa9a0, store: [table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:43,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/863dcdda3ffe40198d91c3f7a06fa9a0 is 175, key is test_row_0/A:col10/1731712663356/Put/seqid=0 2024-11-15T23:17:43,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742363_1539 (size=39549) 2024-11-15T23:17:43,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T23:17:43,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712723495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:43,497 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:43,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T23:17:43,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:43,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:43,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:43,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:43,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:43,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:43,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:43,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712723505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:43,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:43,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712723506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:43,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:43,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712723508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:43,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:43,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712723508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:43,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:43,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:43,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:43,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:43,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T23:17:43,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:43,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:43,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:43,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:43,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T23:17:43,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712723705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:43,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712723716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:43,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:43,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712723716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:43,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:43,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712723716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:43,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:43,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712723716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:43,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:43,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:43,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T23:17:43,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:43,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:43,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:43,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:43,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:43,814 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/863dcdda3ffe40198d91c3f7a06fa9a0 2024-11-15T23:17:43,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/f134c955845c43f68e65379e668948c7 is 50, key is test_row_0/B:col10/1731712663356/Put/seqid=0 2024-11-15T23:17:43,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742364_1540 (size=12001) 2024-11-15T23:17:43,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T23:17:43,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:43,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:43,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:43,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:43,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T23:17:44,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712724015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712724025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712724025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712724025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712724025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T23:17:44,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:44,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:44,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:44,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:44,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:44,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/f134c955845c43f68e65379e668948c7 2024-11-15T23:17:44,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T23:17:44,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:44,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:44,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:44,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:44,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:44,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/aa8b04c838ae4c93aea5d36d248106a2 is 50, key is test_row_0/C:col10/1731712663356/Put/seqid=0 2024-11-15T23:17:44,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742365_1541 (size=12001) 2024-11-15T23:17:44,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/aa8b04c838ae4c93aea5d36d248106a2 2024-11-15T23:17:44,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/863dcdda3ffe40198d91c3f7a06fa9a0 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/863dcdda3ffe40198d91c3f7a06fa9a0 2024-11-15T23:17:44,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/863dcdda3ffe40198d91c3f7a06fa9a0, entries=200, sequenceid=17, filesize=38.6 K 2024-11-15T23:17:44,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/f134c955845c43f68e65379e668948c7 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/f134c955845c43f68e65379e668948c7 2024-11-15T23:17:44,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/f134c955845c43f68e65379e668948c7, entries=150, sequenceid=17, filesize=11.7 K 2024-11-15T23:17:44,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/aa8b04c838ae4c93aea5d36d248106a2 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/aa8b04c838ae4c93aea5d36d248106a2 2024-11-15T23:17:44,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/aa8b04c838ae4c93aea5d36d248106a2, entries=150, sequenceid=17, filesize=11.7 K 2024-11-15T23:17:44,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 06815b48d79c548c6fc12bf365c2f5fd in 958ms, sequenceid=17, compaction requested=false 2024-11-15T23:17:44,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:44,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T23:17:44,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:44,423 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 06815b48d79c548c6fc12bf365c2f5fd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T23:17:44,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=A 2024-11-15T23:17:44,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:44,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=B 2024-11-15T23:17:44,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:44,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=C 2024-11-15T23:17:44,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:44,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b91e1f70c9f241a3a5741f69610a6177_06815b48d79c548c6fc12bf365c2f5fd is 50, key is test_row_0/A:col10/1731712663390/Put/seqid=0 2024-11-15T23:17:44,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742366_1542 (size=12154) 2024-11-15T23:17:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T23:17:44,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:44,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712724539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712724540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712724542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712724540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712724546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712724655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712724655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712724655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712724656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712724656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:44,834 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b91e1f70c9f241a3a5741f69610a6177_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b91e1f70c9f241a3a5741f69610a6177_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:44,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/c6d67fbe7a994cc9bf5a3de67482ada0, store: [table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:44,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/c6d67fbe7a994cc9bf5a3de67482ada0 is 175, key is test_row_0/A:col10/1731712663390/Put/seqid=0 2024-11-15T23:17:44,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742367_1543 (size=30955) 2024-11-15T23:17:44,840 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=37, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/c6d67fbe7a994cc9bf5a3de67482ada0 2024-11-15T23:17:44,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/f691d2301e4340b690b835eff6fe85c9 is 50, key is test_row_0/B:col10/1731712663390/Put/seqid=0 2024-11-15T23:17:44,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742368_1544 (size=12001) 2024-11-15T23:17:44,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712724865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712724866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712724866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712724866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712724866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:44,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:44,970 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T23:17:45,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712725175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:45,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712725175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:45,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712725175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:45,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712725176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712725176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:45,249 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/f691d2301e4340b690b835eff6fe85c9 2024-11-15T23:17:45,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/36bbc13195534ec388ce89e032c19c79 is 50, key is test_row_0/C:col10/1731712663390/Put/seqid=0 2024-11-15T23:17:45,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742369_1545 (size=12001) 2024-11-15T23:17:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T23:17:45,663 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/36bbc13195534ec388ce89e032c19c79 2024-11-15T23:17:45,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/c6d67fbe7a994cc9bf5a3de67482ada0 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/c6d67fbe7a994cc9bf5a3de67482ada0 2024-11-15T23:17:45,669 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/c6d67fbe7a994cc9bf5a3de67482ada0, entries=150, sequenceid=37, filesize=30.2 K 2024-11-15T23:17:45,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/f691d2301e4340b690b835eff6fe85c9 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/f691d2301e4340b690b835eff6fe85c9 2024-11-15T23:17:45,673 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/f691d2301e4340b690b835eff6fe85c9, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T23:17:45,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/36bbc13195534ec388ce89e032c19c79 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/36bbc13195534ec388ce89e032c19c79 2024-11-15T23:17:45,676 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/36bbc13195534ec388ce89e032c19c79, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T23:17:45,677 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 06815b48d79c548c6fc12bf365c2f5fd in 1254ms, sequenceid=37, compaction requested=false 2024-11-15T23:17:45,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:45,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:45,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-15T23:17:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-15T23:17:45,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-15T23:17:45,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3310 sec 2024-11-15T23:17:45,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.3360 sec 2024-11-15T23:17:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:45,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06815b48d79c548c6fc12bf365c2f5fd 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T23:17:45,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=A 2024-11-15T23:17:45,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:45,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=B 2024-11-15T23:17:45,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:45,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=C 2024-11-15T23:17:45,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:45,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c0c46a2d8290401ab9cdab186f40ed9e_06815b48d79c548c6fc12bf365c2f5fd is 50, key is test_row_0/A:col10/1731712665688/Put/seqid=0 2024-11-15T23:17:45,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742370_1546 (size=12154) 2024-11-15T23:17:45,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712725703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:45,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712725704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:45,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712725705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:45,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712725706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:45,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:45,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712725707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:45,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:45,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712725815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:45,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712725815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712725815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712725815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712725815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:46,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712726025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:46,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712726025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712726025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:46,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712726025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712726026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:46,097 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:46,099 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c0c46a2d8290401ab9cdab186f40ed9e_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c0c46a2d8290401ab9cdab186f40ed9e_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:46,100 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/376af1ed0cff4b188f158768566ee302, store: [table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:46,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/376af1ed0cff4b188f158768566ee302 is 175, key is test_row_0/A:col10/1731712665688/Put/seqid=0 2024-11-15T23:17:46,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742371_1547 (size=30955) 2024-11-15T23:17:46,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712726335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:46,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712726335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:46,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712726335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:46,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712726335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712726335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:46,503 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/376af1ed0cff4b188f158768566ee302 2024-11-15T23:17:46,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/d2f238ffccce4a1eb90ac81ed245ade1 is 50, key is test_row_0/B:col10/1731712665688/Put/seqid=0 2024-11-15T23:17:46,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742372_1548 (size=12001) 2024-11-15T23:17:46,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712726845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:46,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712726845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:46,848 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,848 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,848 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712726845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:46,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712726845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712726845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:46,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/d2f238ffccce4a1eb90ac81ed245ade1 2024-11-15T23:17:46,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/bbae5312d3cc4660a926927ced6e4289 is 50, key is test_row_0/C:col10/1731712665688/Put/seqid=0 2024-11-15T23:17:46,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742373_1549 (size=12001) 2024-11-15T23:17:46,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/bbae5312d3cc4660a926927ced6e4289 2024-11-15T23:17:46,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/376af1ed0cff4b188f158768566ee302 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/376af1ed0cff4b188f158768566ee302 2024-11-15T23:17:46,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/376af1ed0cff4b188f158768566ee302, entries=150, sequenceid=51, filesize=30.2 K 2024-11-15T23:17:46,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/d2f238ffccce4a1eb90ac81ed245ade1 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/d2f238ffccce4a1eb90ac81ed245ade1 2024-11-15T23:17:46,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/d2f238ffccce4a1eb90ac81ed245ade1, entries=150, sequenceid=51, filesize=11.7 K 2024-11-15T23:17:46,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/bbae5312d3cc4660a926927ced6e4289 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/bbae5312d3cc4660a926927ced6e4289 2024-11-15T23:17:46,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/bbae5312d3cc4660a926927ced6e4289, entries=150, sequenceid=51, filesize=11.7 K 2024-11-15T23:17:46,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 06815b48d79c548c6fc12bf365c2f5fd in 1247ms, sequenceid=51, compaction requested=true 2024-11-15T23:17:46,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:46,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06815b48d79c548c6fc12bf365c2f5fd:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:17:46,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:46,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06815b48d79c548c6fc12bf365c2f5fd:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:17:46,936 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:46,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:46,936 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:46,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06815b48d79c548c6fc12bf365c2f5fd:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:17:46,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:46,937 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:46,937 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:46,937 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 06815b48d79c548c6fc12bf365c2f5fd/A is initiating minor compaction (all files) 2024-11-15T23:17:46,937 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 06815b48d79c548c6fc12bf365c2f5fd/B is initiating minor compaction (all files) 2024-11-15T23:17:46,937 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06815b48d79c548c6fc12bf365c2f5fd/A in TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:46,937 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06815b48d79c548c6fc12bf365c2f5fd/B in TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:46,937 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/f134c955845c43f68e65379e668948c7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/f691d2301e4340b690b835eff6fe85c9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/d2f238ffccce4a1eb90ac81ed245ade1] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp, totalSize=35.2 K 2024-11-15T23:17:46,937 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/863dcdda3ffe40198d91c3f7a06fa9a0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/c6d67fbe7a994cc9bf5a3de67482ada0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/376af1ed0cff4b188f158768566ee302] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp, totalSize=99.1 K 2024-11-15T23:17:46,937 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:46,937 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. files: [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/863dcdda3ffe40198d91c3f7a06fa9a0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/c6d67fbe7a994cc9bf5a3de67482ada0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/376af1ed0cff4b188f158768566ee302] 2024-11-15T23:17:46,938 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting f134c955845c43f68e65379e668948c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731712663349 2024-11-15T23:17:46,938 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 863dcdda3ffe40198d91c3f7a06fa9a0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731712663349 2024-11-15T23:17:46,938 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting f691d2301e4340b690b835eff6fe85c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731712663382 2024-11-15T23:17:46,938 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6d67fbe7a994cc9bf5a3de67482ada0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731712663382 2024-11-15T23:17:46,939 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting d2f238ffccce4a1eb90ac81ed245ade1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731712664536 2024-11-15T23:17:46,939 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 376af1ed0cff4b188f158768566ee302, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731712664536 2024-11-15T23:17:46,944 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06815b48d79c548c6fc12bf365c2f5fd#B#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:46,944 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:46,945 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/e7d58a488fdb4e3a9951dea8c2420a62 is 50, key is test_row_0/B:col10/1731712665688/Put/seqid=0 2024-11-15T23:17:46,946 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115317022d8d9a944fcab1d9bfac9050d26_06815b48d79c548c6fc12bf365c2f5fd store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:46,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742374_1550 (size=12104) 2024-11-15T23:17:46,948 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115317022d8d9a944fcab1d9bfac9050d26_06815b48d79c548c6fc12bf365c2f5fd, store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:46,948 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115317022d8d9a944fcab1d9bfac9050d26_06815b48d79c548c6fc12bf365c2f5fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:46,959 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/e7d58a488fdb4e3a9951dea8c2420a62 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/e7d58a488fdb4e3a9951dea8c2420a62 2024-11-15T23:17:46,964 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06815b48d79c548c6fc12bf365c2f5fd/B of 06815b48d79c548c6fc12bf365c2f5fd into e7d58a488fdb4e3a9951dea8c2420a62(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:46,964 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:46,964 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., storeName=06815b48d79c548c6fc12bf365c2f5fd/B, priority=13, startTime=1731712666936; duration=0sec 2024-11-15T23:17:46,964 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:46,964 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06815b48d79c548c6fc12bf365c2f5fd:B 2024-11-15T23:17:46,964 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:17:46,965 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:17:46,966 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 06815b48d79c548c6fc12bf365c2f5fd/C is initiating minor compaction (all files) 2024-11-15T23:17:46,966 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06815b48d79c548c6fc12bf365c2f5fd/C in TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:46,966 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/aa8b04c838ae4c93aea5d36d248106a2, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/36bbc13195534ec388ce89e032c19c79, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/bbae5312d3cc4660a926927ced6e4289] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp, totalSize=35.2 K 2024-11-15T23:17:46,966 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting aa8b04c838ae4c93aea5d36d248106a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731712663349 2024-11-15T23:17:46,967 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 36bbc13195534ec388ce89e032c19c79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731712663382 2024-11-15T23:17:46,967 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting bbae5312d3cc4660a926927ced6e4289, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731712664536 2024-11-15T23:17:46,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742375_1551 (size=4469) 2024-11-15T23:17:46,980 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06815b48d79c548c6fc12bf365c2f5fd#C#compaction#472 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:46,980 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/f9135a7606ad4cc8affefcbcfab61532 is 50, key is test_row_0/C:col10/1731712665688/Put/seqid=0 2024-11-15T23:17:46,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742376_1552 (size=12104) 2024-11-15T23:17:47,382 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06815b48d79c548c6fc12bf365c2f5fd#A#compaction#471 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:47,382 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/ee090792f9794d97a76ed32fe4ac71e2 is 175, key is test_row_0/A:col10/1731712665688/Put/seqid=0 2024-11-15T23:17:47,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742377_1553 (size=31058) 2024-11-15T23:17:47,388 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/f9135a7606ad4cc8affefcbcfab61532 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/f9135a7606ad4cc8affefcbcfab61532 2024-11-15T23:17:47,392 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06815b48d79c548c6fc12bf365c2f5fd/C of 06815b48d79c548c6fc12bf365c2f5fd into f9135a7606ad4cc8affefcbcfab61532(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:47,392 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:47,392 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., storeName=06815b48d79c548c6fc12bf365c2f5fd/C, priority=13, startTime=1731712666936; duration=0sec 2024-11-15T23:17:47,392 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:47,392 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06815b48d79c548c6fc12bf365c2f5fd:C 2024-11-15T23:17:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T23:17:47,515 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:47,516 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:17:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-15T23:17:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T23:17:47,517 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:17:47,518 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:17:47,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:17:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T23:17:47,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T23:17:47,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:47,670 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 06815b48d79c548c6fc12bf365c2f5fd 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T23:17:47,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=A 2024-11-15T23:17:47,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:47,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=B 2024-11-15T23:17:47,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:47,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=C 2024-11-15T23:17:47,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:47,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115395bca1000594cb9b6d465a9b15c5fa2_06815b48d79c548c6fc12bf365c2f5fd is 50, key is test_row_0/A:col10/1731712665700/Put/seqid=0 2024-11-15T23:17:47,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742378_1554 (size=12154) 2024-11-15T23:17:47,792 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/ee090792f9794d97a76ed32fe4ac71e2 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/ee090792f9794d97a76ed32fe4ac71e2 2024-11-15T23:17:47,795 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06815b48d79c548c6fc12bf365c2f5fd/A of 06815b48d79c548c6fc12bf365c2f5fd into ee090792f9794d97a76ed32fe4ac71e2(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:47,795 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:47,795 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., storeName=06815b48d79c548c6fc12bf365c2f5fd/A, priority=13, startTime=1731712666936; duration=0sec 2024-11-15T23:17:47,795 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:47,795 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06815b48d79c548c6fc12bf365c2f5fd:A 2024-11-15T23:17:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T23:17:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:47,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:47,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712727870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:47,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712727870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:47,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712727872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:47,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712727875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:47,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:47,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712727876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:47,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:47,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:47,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712727985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:47,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712727985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:47,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712727985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712727986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:47,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712727986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:48,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:48,087 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115395bca1000594cb9b6d465a9b15c5fa2_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115395bca1000594cb9b6d465a9b15c5fa2_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:48,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/ab983dae524d400d8afabb9cbc4e4d69, store: [table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:48,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/ab983dae524d400d8afabb9cbc4e4d69 is 175, key is test_row_0/A:col10/1731712665700/Put/seqid=0 2024-11-15T23:17:48,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742379_1555 (size=30955) 2024-11-15T23:17:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T23:17:48,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712728196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:48,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712728196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:48,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712728196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:48,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712728196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:48,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712728196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:48,492 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=71, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/ab983dae524d400d8afabb9cbc4e4d69 2024-11-15T23:17:48,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/004b8216c6504b3e84590888cfc296f5 is 50, key is test_row_0/B:col10/1731712665700/Put/seqid=0 2024-11-15T23:17:48,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742380_1556 (size=12001) 2024-11-15T23:17:48,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712728506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:48,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712728506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712728506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712728506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:48,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:48,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712728507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:48,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:48,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T23:17:48,907 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/004b8216c6504b3e84590888cfc296f5 2024-11-15T23:17:48,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/ef60a25c80294d60961c5637f8405b3b is 50, key is test_row_0/C:col10/1731712665700/Put/seqid=0 2024-11-15T23:17:48,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742381_1557 (size=12001) 2024-11-15T23:17:49,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712729015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:49,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712729015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:49,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712729015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:49,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712729015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:49,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712729016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:49,317 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/ef60a25c80294d60961c5637f8405b3b 2024-11-15T23:17:49,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/ab983dae524d400d8afabb9cbc4e4d69 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/ab983dae524d400d8afabb9cbc4e4d69 2024-11-15T23:17:49,323 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/ab983dae524d400d8afabb9cbc4e4d69, entries=150, sequenceid=71, filesize=30.2 K 2024-11-15T23:17:49,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/004b8216c6504b3e84590888cfc296f5 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/004b8216c6504b3e84590888cfc296f5 2024-11-15T23:17:49,326 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/004b8216c6504b3e84590888cfc296f5, entries=150, sequenceid=71, filesize=11.7 K 2024-11-15T23:17:49,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/ef60a25c80294d60961c5637f8405b3b as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/ef60a25c80294d60961c5637f8405b3b 2024-11-15T23:17:49,329 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/ef60a25c80294d60961c5637f8405b3b, entries=150, sequenceid=71, filesize=11.7 K 2024-11-15T23:17:49,330 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 06815b48d79c548c6fc12bf365c2f5fd in 1660ms, sequenceid=71, compaction requested=false 2024-11-15T23:17:49,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:49,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:49,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-15T23:17:49,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-15T23:17:49,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-15T23:17:49,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8130 sec 2024-11-15T23:17:49,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.8170 sec 2024-11-15T23:17:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T23:17:49,675 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:49,675 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:17:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-15T23:17:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T23:17:49,679 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:17:49,680 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:17:49,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:17:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T23:17:49,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T23:17:49,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:49,832 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 06815b48d79c548c6fc12bf365c2f5fd 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T23:17:49,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=A 2024-11-15T23:17:49,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:49,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=B 2024-11-15T23:17:49,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:49,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=C 2024-11-15T23:17:49,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:49,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b33e85ac978b409196ae8b048de6e6b7_06815b48d79c548c6fc12bf365c2f5fd is 50, key is test_row_0/A:col10/1731712667870/Put/seqid=0 2024-11-15T23:17:49,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742382_1558 (size=12154) 2024-11-15T23:17:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T23:17:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:50,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:50,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712730084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712730085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712730086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712730090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712730090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712730195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712730195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712730195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712730195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712730205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:50,243 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b33e85ac978b409196ae8b048de6e6b7_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b33e85ac978b409196ae8b048de6e6b7_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:50,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/2ddaed68bc654196a31408ba5f004707, store: [table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:50,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/2ddaed68bc654196a31408ba5f004707 is 175, key is test_row_0/A:col10/1731712667870/Put/seqid=0 2024-11-15T23:17:50,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742383_1559 (size=30955) 2024-11-15T23:17:50,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T23:17:50,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712730405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712730405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712730405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712730406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712730415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,647 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/2ddaed68bc654196a31408ba5f004707 2024-11-15T23:17:50,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/bb8a8e0c5bcc48cabcd03f1f6179b240 is 50, key is test_row_0/B:col10/1731712667870/Put/seqid=0 2024-11-15T23:17:50,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742384_1560 (size=12001) 2024-11-15T23:17:50,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712730715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712730715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712730715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712730715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712730725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T23:17:51,065 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/bb8a8e0c5bcc48cabcd03f1f6179b240 2024-11-15T23:17:51,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/e542ba34d5704e03b6c34aaa204d288d is 50, key is test_row_0/C:col10/1731712667870/Put/seqid=0 2024-11-15T23:17:51,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742385_1561 (size=12001) 2024-11-15T23:17:51,074 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/e542ba34d5704e03b6c34aaa204d288d 2024-11-15T23:17:51,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/2ddaed68bc654196a31408ba5f004707 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/2ddaed68bc654196a31408ba5f004707 2024-11-15T23:17:51,084 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/2ddaed68bc654196a31408ba5f004707, entries=150, sequenceid=86, filesize=30.2 K 2024-11-15T23:17:51,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/bb8a8e0c5bcc48cabcd03f1f6179b240 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/bb8a8e0c5bcc48cabcd03f1f6179b240 2024-11-15T23:17:51,088 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/bb8a8e0c5bcc48cabcd03f1f6179b240, entries=150, sequenceid=86, filesize=11.7 K 2024-11-15T23:17:51,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/e542ba34d5704e03b6c34aaa204d288d as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/e542ba34d5704e03b6c34aaa204d288d 2024-11-15T23:17:51,092 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/e542ba34d5704e03b6c34aaa204d288d, entries=150, sequenceid=86, filesize=11.7 K 2024-11-15T23:17:51,094 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 06815b48d79c548c6fc12bf365c2f5fd in 1262ms, sequenceid=86, compaction requested=true 2024-11-15T23:17:51,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:51,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:51,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-15T23:17:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-15T23:17:51,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-15T23:17:51,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-11-15T23:17:51,100 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.4230 sec 2024-11-15T23:17:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:51,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06815b48d79c548c6fc12bf365c2f5fd 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T23:17:51,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=A 2024-11-15T23:17:51,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:51,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=B 2024-11-15T23:17:51,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:51,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=C 2024-11-15T23:17:51,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:51,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115df000f2c71664cba972e6906438c0dcd_06815b48d79c548c6fc12bf365c2f5fd is 50, key is test_row_0/A:col10/1731712670083/Put/seqid=0 2024-11-15T23:17:51,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742386_1562 (size=14594) 2024-11-15T23:17:51,239 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:51,242 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115df000f2c71664cba972e6906438c0dcd_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115df000f2c71664cba972e6906438c0dcd_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:51,243 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/e101d612c8a243bc9f1025fbd0dc3278, store: [table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:51,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/e101d612c8a243bc9f1025fbd0dc3278 is 175, key is test_row_0/A:col10/1731712670083/Put/seqid=0 2024-11-15T23:17:51,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712731238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742387_1563 (size=39549) 2024-11-15T23:17:51,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712731241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,247 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/e101d612c8a243bc9f1025fbd0dc3278 2024-11-15T23:17:51,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712731245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712731246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712731247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/0e717e5e19ce451b87afe7da8fe70ad7 is 50, key is test_row_0/B:col10/1731712670083/Put/seqid=0 2024-11-15T23:17:51,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742388_1564 (size=12001) 2024-11-15T23:17:51,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712731355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712731355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712731355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712731355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712731365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712731565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712731565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712731565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712731565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712731575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/0e717e5e19ce451b87afe7da8fe70ad7 2024-11-15T23:17:51,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/bfa00165989b4b9c9eda67a7731768e2 is 50, key is test_row_0/C:col10/1731712670083/Put/seqid=0 2024-11-15T23:17:51,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742389_1565 (size=12001) 2024-11-15T23:17:51,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/bfa00165989b4b9c9eda67a7731768e2 2024-11-15T23:17:51,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/e101d612c8a243bc9f1025fbd0dc3278 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/e101d612c8a243bc9f1025fbd0dc3278 2024-11-15T23:17:51,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/e101d612c8a243bc9f1025fbd0dc3278, entries=200, sequenceid=105, filesize=38.6 K 2024-11-15T23:17:51,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/0e717e5e19ce451b87afe7da8fe70ad7 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/0e717e5e19ce451b87afe7da8fe70ad7 2024-11-15T23:17:51,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/0e717e5e19ce451b87afe7da8fe70ad7, entries=150, sequenceid=105, filesize=11.7 K 2024-11-15T23:17:51,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/bfa00165989b4b9c9eda67a7731768e2 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/bfa00165989b4b9c9eda67a7731768e2 2024-11-15T23:17:51,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/bfa00165989b4b9c9eda67a7731768e2, entries=150, sequenceid=105, filesize=11.7 K 2024-11-15T23:17:51,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 06815b48d79c548c6fc12bf365c2f5fd in 485ms, sequenceid=105, compaction requested=true 2024-11-15T23:17:51,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:51,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06815b48d79c548c6fc12bf365c2f5fd:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:17:51,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:51,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06815b48d79c548c6fc12bf365c2f5fd:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:17:51,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:51,714 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:17:51,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06815b48d79c548c6fc12bf365c2f5fd:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:17:51,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T23:17:51,714 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:17:51,715 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:17:51,715 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 06815b48d79c548c6fc12bf365c2f5fd/A is initiating minor compaction (all files) 2024-11-15T23:17:51,715 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06815b48d79c548c6fc12bf365c2f5fd/A in TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:51,715 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/ee090792f9794d97a76ed32fe4ac71e2, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/ab983dae524d400d8afabb9cbc4e4d69, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/2ddaed68bc654196a31408ba5f004707, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/e101d612c8a243bc9f1025fbd0dc3278] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp, totalSize=129.4 K 2024-11-15T23:17:51,715 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:51,715 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. files: [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/ee090792f9794d97a76ed32fe4ac71e2, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/ab983dae524d400d8afabb9cbc4e4d69, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/2ddaed68bc654196a31408ba5f004707, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/e101d612c8a243bc9f1025fbd0dc3278] 2024-11-15T23:17:51,715 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:17:51,715 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 06815b48d79c548c6fc12bf365c2f5fd/B is initiating minor compaction (all files) 2024-11-15T23:17:51,715 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06815b48d79c548c6fc12bf365c2f5fd/B in TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:51,715 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/e7d58a488fdb4e3a9951dea8c2420a62, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/004b8216c6504b3e84590888cfc296f5, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/bb8a8e0c5bcc48cabcd03f1f6179b240, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/0e717e5e19ce451b87afe7da8fe70ad7] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp, totalSize=47.0 K 2024-11-15T23:17:51,715 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee090792f9794d97a76ed32fe4ac71e2, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731712664536 2024-11-15T23:17:51,715 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting e7d58a488fdb4e3a9951dea8c2420a62, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731712664536 2024-11-15T23:17:51,716 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab983dae524d400d8afabb9cbc4e4d69, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1731712665700 2024-11-15T23:17:51,716 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 004b8216c6504b3e84590888cfc296f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1731712665700 2024-11-15T23:17:51,716 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ddaed68bc654196a31408ba5f004707, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731712667865 2024-11-15T23:17:51,717 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting bb8a8e0c5bcc48cabcd03f1f6179b240, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731712667865 2024-11-15T23:17:51,717 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting e101d612c8a243bc9f1025fbd0dc3278, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731712670083 2024-11-15T23:17:51,718 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e717e5e19ce451b87afe7da8fe70ad7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731712670083 2024-11-15T23:17:51,726 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:51,728 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06815b48d79c548c6fc12bf365c2f5fd#B#compaction#482 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:51,728 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/9c8545fcc3ca4aa49477316821359a1a is 50, key is test_row_0/B:col10/1731712670083/Put/seqid=0 2024-11-15T23:17:51,733 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411155e3f256951cc4ae48dc48f04a5998804_06815b48d79c548c6fc12bf365c2f5fd store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:51,735 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411155e3f256951cc4ae48dc48f04a5998804_06815b48d79c548c6fc12bf365c2f5fd, store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:51,735 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155e3f256951cc4ae48dc48f04a5998804_06815b48d79c548c6fc12bf365c2f5fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:51,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742390_1566 (size=12241) 2024-11-15T23:17:51,770 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/9c8545fcc3ca4aa49477316821359a1a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/9c8545fcc3ca4aa49477316821359a1a 2024-11-15T23:17:51,774 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 06815b48d79c548c6fc12bf365c2f5fd/B of 06815b48d79c548c6fc12bf365c2f5fd into 9c8545fcc3ca4aa49477316821359a1a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:51,774 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:51,774 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., storeName=06815b48d79c548c6fc12bf365c2f5fd/B, priority=12, startTime=1731712671713; duration=0sec 2024-11-15T23:17:51,774 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:51,774 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06815b48d79c548c6fc12bf365c2f5fd:B 2024-11-15T23:17:51,774 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:17:51,776 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:17:51,776 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 06815b48d79c548c6fc12bf365c2f5fd/C is initiating minor compaction (all files) 2024-11-15T23:17:51,776 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06815b48d79c548c6fc12bf365c2f5fd/C in TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:51,776 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/f9135a7606ad4cc8affefcbcfab61532, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/ef60a25c80294d60961c5637f8405b3b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/e542ba34d5704e03b6c34aaa204d288d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/bfa00165989b4b9c9eda67a7731768e2] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp, totalSize=47.0 K 2024-11-15T23:17:51,777 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting f9135a7606ad4cc8affefcbcfab61532, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731712664536 2024-11-15T23:17:51,777 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting ef60a25c80294d60961c5637f8405b3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1731712665700 2024-11-15T23:17:51,777 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting e542ba34d5704e03b6c34aaa204d288d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731712667865 2024-11-15T23:17:51,777 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting bfa00165989b4b9c9eda67a7731768e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731712670083 2024-11-15T23:17:51,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742391_1567 (size=4469) 2024-11-15T23:17:51,779 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06815b48d79c548c6fc12bf365c2f5fd#A#compaction#483 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:51,780 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/f32e8e274cde427689d9c5492b0ac52d is 175, key is test_row_0/A:col10/1731712670083/Put/seqid=0 2024-11-15T23:17:51,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742392_1568 (size=31195) 2024-11-15T23:17:51,798 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06815b48d79c548c6fc12bf365c2f5fd#C#compaction#484 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:51,798 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/ac9caa584ceb4f60b9f39a2904d62ab0 is 50, key is test_row_0/C:col10/1731712670083/Put/seqid=0 2024-11-15T23:17:51,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742393_1569 (size=12241) 2024-11-15T23:17:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T23:17:51,826 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:51,826 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:17:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-15T23:17:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T23:17:51,829 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:17:51,829 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:17:51,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:17:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:51,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06815b48d79c548c6fc12bf365c2f5fd 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T23:17:51,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=A 2024-11-15T23:17:51,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:51,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=B 2024-11-15T23:17:51,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:51,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=C 2024-11-15T23:17:51,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:51,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115355b63deaf8049b1946427e83a38bec6_06815b48d79c548c6fc12bf365c2f5fd is 50, key is test_row_0/A:col10/1731712671876/Put/seqid=0 2024-11-15T23:17:51,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742394_1570 (size=14594) 2024-11-15T23:17:51,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712731894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712731894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712731896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712731896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712731897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T23:17:51,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T23:17:51,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:51,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:51,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:51,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:51,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:52,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712732005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712732005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:52,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712732005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:52,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712732005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:52,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712732005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:52,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T23:17:52,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:52,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:52,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:52,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:52,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T23:17:52,193 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/f32e8e274cde427689d9c5492b0ac52d as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/f32e8e274cde427689d9c5492b0ac52d 2024-11-15T23:17:52,196 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 06815b48d79c548c6fc12bf365c2f5fd/A of 06815b48d79c548c6fc12bf365c2f5fd into f32e8e274cde427689d9c5492b0ac52d(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:52,196 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:52,196 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., storeName=06815b48d79c548c6fc12bf365c2f5fd/A, priority=12, startTime=1731712671713; duration=0sec 2024-11-15T23:17:52,196 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:52,196 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06815b48d79c548c6fc12bf365c2f5fd:A 2024-11-15T23:17:52,208 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/ac9caa584ceb4f60b9f39a2904d62ab0 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/ac9caa584ceb4f60b9f39a2904d62ab0 2024-11-15T23:17:52,212 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 06815b48d79c548c6fc12bf365c2f5fd/C of 06815b48d79c548c6fc12bf365c2f5fd into ac9caa584ceb4f60b9f39a2904d62ab0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:52,212 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:52,212 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., storeName=06815b48d79c548c6fc12bf365c2f5fd/C, priority=12, startTime=1731712671714; duration=0sec 2024-11-15T23:17:52,212 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:52,212 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06815b48d79c548c6fc12bf365c2f5fd:C 2024-11-15T23:17:52,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:52,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712732216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712732215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:52,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:52,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712732216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712732216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712732216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:52,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T23:17:52,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:52,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:52,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:52,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:52,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:52,290 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:52,293 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115355b63deaf8049b1946427e83a38bec6_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115355b63deaf8049b1946427e83a38bec6_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:52,294 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/f1a5c7dd95fb49d1ac9bcba6f456da34, store: [table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:52,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/f1a5c7dd95fb49d1ac9bcba6f456da34 is 175, key is test_row_0/A:col10/1731712671876/Put/seqid=0 2024-11-15T23:17:52,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742395_1571 (size=39549) 2024-11-15T23:17:52,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T23:17:52,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:52,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:52,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:52,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:52,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T23:17:52,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:52,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712732525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712732526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:52,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:52,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:52,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712732526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:52,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:52,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712732526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:52,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:52,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712732526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:52,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:52,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:52,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T23:17:52,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:52,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:52,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:52,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:52,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:52,703 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/f1a5c7dd95fb49d1ac9bcba6f456da34 2024-11-15T23:17:52,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/b44008aab37f4aceb418a3146b382175 is 50, key is test_row_0/B:col10/1731712671876/Put/seqid=0 2024-11-15T23:17:52,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742396_1572 (size=12001) 2024-11-15T23:17:52,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T23:17:52,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:52,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:52,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:52,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:52,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:52,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T23:17:52,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:52,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:52,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:52,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:52,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T23:17:53,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:53,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712733036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:53,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:53,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:53,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712733036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:53,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712733036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:53,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:53,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:53,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712733036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:53,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712733036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:53,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T23:17:53,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:53,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:53,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:53,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:53,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:53,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:53,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/b44008aab37f4aceb418a3146b382175 2024-11-15T23:17:53,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/3110e18872ce4a2080c46c78844b0a48 is 50, key is test_row_0/C:col10/1731712671876/Put/seqid=0 2024-11-15T23:17:53,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742397_1573 (size=12001) 2024-11-15T23:17:53,202 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T23:17:53,202 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T23:17:53,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T23:17:53,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:53,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:53,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:53,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:53,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:53,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:53,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T23:17:53,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:53,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:53,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:53,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:53,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:53,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T23:17:53,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:53,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:53,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:53,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:53,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:53,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:53,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/3110e18872ce4a2080c46c78844b0a48 2024-11-15T23:17:53,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/f1a5c7dd95fb49d1ac9bcba6f456da34 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/f1a5c7dd95fb49d1ac9bcba6f456da34 2024-11-15T23:17:53,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/f1a5c7dd95fb49d1ac9bcba6f456da34, entries=200, sequenceid=121, filesize=38.6 K 2024-11-15T23:17:53,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/b44008aab37f4aceb418a3146b382175 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/b44008aab37f4aceb418a3146b382175 2024-11-15T23:17:53,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/b44008aab37f4aceb418a3146b382175, entries=150, sequenceid=121, filesize=11.7 K 2024-11-15T23:17:53,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/3110e18872ce4a2080c46c78844b0a48 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/3110e18872ce4a2080c46c78844b0a48 2024-11-15T23:17:53,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/3110e18872ce4a2080c46c78844b0a48, entries=150, sequenceid=121, filesize=11.7 K 2024-11-15T23:17:53,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 06815b48d79c548c6fc12bf365c2f5fd in 1686ms, sequenceid=121, compaction requested=false 2024-11-15T23:17:53,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:53,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T23:17:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:53,662 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 06815b48d79c548c6fc12bf365c2f5fd 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T23:17:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=A 2024-11-15T23:17:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=B 2024-11-15T23:17:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=C 2024-11-15T23:17:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:53,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157c5e491880fe4e879d8bea8f82970706_06815b48d79c548c6fc12bf365c2f5fd is 50, key is test_row_0/A:col10/1731712671887/Put/seqid=0 2024-11-15T23:17:53,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742398_1574 (size=12304) 2024-11-15T23:17:53,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:53,685 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157c5e491880fe4e879d8bea8f82970706_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157c5e491880fe4e879d8bea8f82970706_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:53,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/9c2d0ff4a2f84a0d9945e9b9e4afe32e, store: [table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:53,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/9c2d0ff4a2f84a0d9945e9b9e4afe32e is 175, key is test_row_0/A:col10/1731712671887/Put/seqid=0 2024-11-15T23:17:53,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742399_1575 (size=31105) 2024-11-15T23:17:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T23:17:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:54,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:54,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712734062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712734063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712734063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712734065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712734065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,091 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/9c2d0ff4a2f84a0d9945e9b9e4afe32e 2024-11-15T23:17:54,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/7c82e93185774a5fb9bfae8333bb44af is 50, key is test_row_0/B:col10/1731712671887/Put/seqid=0 2024-11-15T23:17:54,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742400_1576 (size=12151) 2024-11-15T23:17:54,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712734176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712734176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712734176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712734176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712734176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712734385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712734385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712734385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712734385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712734386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,518 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/7c82e93185774a5fb9bfae8333bb44af 2024-11-15T23:17:54,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/9241baf91f564beca893158bc4a2b98d is 50, key is test_row_0/C:col10/1731712671887/Put/seqid=0 2024-11-15T23:17:54,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742401_1577 (size=12151) 2024-11-15T23:17:54,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712734695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712734696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712734696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712734696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:54,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712734696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:54,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:54,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:54,938 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/9241baf91f564beca893158bc4a2b98d 2024-11-15T23:17:54,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/9c2d0ff4a2f84a0d9945e9b9e4afe32e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/9c2d0ff4a2f84a0d9945e9b9e4afe32e 2024-11-15T23:17:54,945 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/9c2d0ff4a2f84a0d9945e9b9e4afe32e, entries=150, sequenceid=140, filesize=30.4 K 2024-11-15T23:17:54,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/7c82e93185774a5fb9bfae8333bb44af as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/7c82e93185774a5fb9bfae8333bb44af 2024-11-15T23:17:54,949 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/7c82e93185774a5fb9bfae8333bb44af, entries=150, sequenceid=140, filesize=11.9 K 2024-11-15T23:17:54,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/9241baf91f564beca893158bc4a2b98d as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/9241baf91f564beca893158bc4a2b98d 2024-11-15T23:17:54,958 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/9241baf91f564beca893158bc4a2b98d, entries=150, sequenceid=140, filesize=11.9 K 2024-11-15T23:17:54,960 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 06815b48d79c548c6fc12bf365c2f5fd in 1298ms, sequenceid=140, compaction requested=true 2024-11-15T23:17:54,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:54,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:54,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-15T23:17:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-15T23:17:54,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-15T23:17:54,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1320 sec 2024-11-15T23:17:54,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 3.1400 sec 2024-11-15T23:17:55,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06815b48d79c548c6fc12bf365c2f5fd 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T23:17:55,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=A 2024-11-15T23:17:55,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:55,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=B 2024-11-15T23:17:55,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:55,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=C 2024-11-15T23:17:55,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:55,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:55,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c0780f0816c946908e038809df7ddaeb_06815b48d79c548c6fc12bf365c2f5fd is 50, key is test_row_0/A:col10/1731712674063/Put/seqid=0 2024-11-15T23:17:55,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712735223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712735224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712735225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742402_1578 (size=14794) 2024-11-15T23:17:55,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712735229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712735230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712735336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712735336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712735336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712735345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712735345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712735546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712735546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712735546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712735556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712735556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,634 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:55,639 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c0780f0816c946908e038809df7ddaeb_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c0780f0816c946908e038809df7ddaeb_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:55,640 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/68466811c19b49449285fc2aac90302e, store: [table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:55,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/68466811c19b49449285fc2aac90302e is 175, key is test_row_0/A:col10/1731712674063/Put/seqid=0 2024-11-15T23:17:55,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742403_1579 (size=39749) 2024-11-15T23:17:55,650 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/68466811c19b49449285fc2aac90302e 2024-11-15T23:17:55,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/cf1e7ae212ea4f459932272d96d3f18f is 50, key is test_row_0/B:col10/1731712674063/Put/seqid=0 2024-11-15T23:17:55,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742404_1580 (size=12151) 2024-11-15T23:17:55,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/cf1e7ae212ea4f459932272d96d3f18f 2024-11-15T23:17:55,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/ba9ddff0cd6d4a4db68ccd799d16f7a2 is 50, key is test_row_0/C:col10/1731712674063/Put/seqid=0 2024-11-15T23:17:55,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742405_1581 (size=12151) 2024-11-15T23:17:55,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/ba9ddff0cd6d4a4db68ccd799d16f7a2 2024-11-15T23:17:55,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/68466811c19b49449285fc2aac90302e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/68466811c19b49449285fc2aac90302e 2024-11-15T23:17:55,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/68466811c19b49449285fc2aac90302e, entries=200, sequenceid=157, filesize=38.8 K 2024-11-15T23:17:55,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/cf1e7ae212ea4f459932272d96d3f18f as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/cf1e7ae212ea4f459932272d96d3f18f 2024-11-15T23:17:55,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/cf1e7ae212ea4f459932272d96d3f18f, entries=150, sequenceid=157, filesize=11.9 K 2024-11-15T23:17:55,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/ba9ddff0cd6d4a4db68ccd799d16f7a2 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/ba9ddff0cd6d4a4db68ccd799d16f7a2 2024-11-15T23:17:55,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/ba9ddff0cd6d4a4db68ccd799d16f7a2, entries=150, sequenceid=157, filesize=11.9 K 2024-11-15T23:17:55,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 06815b48d79c548c6fc12bf365c2f5fd in 509ms, sequenceid=157, compaction requested=true 2024-11-15T23:17:55,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:55,719 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:17:55,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06815b48d79c548c6fc12bf365c2f5fd:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:17:55,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:55,720 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141598 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:17:55,721 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:17:55,721 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 06815b48d79c548c6fc12bf365c2f5fd/A is initiating minor compaction (all files) 2024-11-15T23:17:55,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06815b48d79c548c6fc12bf365c2f5fd:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:17:55,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:55,721 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06815b48d79c548c6fc12bf365c2f5fd/A in TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:55,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06815b48d79c548c6fc12bf365c2f5fd:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:17:55,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:55,721 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/f32e8e274cde427689d9c5492b0ac52d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/f1a5c7dd95fb49d1ac9bcba6f456da34, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/9c2d0ff4a2f84a0d9945e9b9e4afe32e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/68466811c19b49449285fc2aac90302e] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp, totalSize=138.3 K 2024-11-15T23:17:55,721 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:55,721 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. files: [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/f32e8e274cde427689d9c5492b0ac52d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/f1a5c7dd95fb49d1ac9bcba6f456da34, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/9c2d0ff4a2f84a0d9945e9b9e4afe32e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/68466811c19b49449285fc2aac90302e] 2024-11-15T23:17:55,722 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting f32e8e274cde427689d9c5492b0ac52d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731712670083 2024-11-15T23:17:55,722 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1a5c7dd95fb49d1ac9bcba6f456da34, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731712671239 2024-11-15T23:17:55,722 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c2d0ff4a2f84a0d9945e9b9e4afe32e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731712671887 2024-11-15T23:17:55,723 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:17:55,723 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 06815b48d79c548c6fc12bf365c2f5fd/B is initiating minor compaction (all files) 2024-11-15T23:17:55,723 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06815b48d79c548c6fc12bf365c2f5fd/B in TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:55,723 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/9c8545fcc3ca4aa49477316821359a1a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/b44008aab37f4aceb418a3146b382175, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/7c82e93185774a5fb9bfae8333bb44af, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/cf1e7ae212ea4f459932272d96d3f18f] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp, totalSize=47.4 K 2024-11-15T23:17:55,723 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68466811c19b49449285fc2aac90302e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731712674055 2024-11-15T23:17:55,723 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c8545fcc3ca4aa49477316821359a1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731712670083 2024-11-15T23:17:55,724 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting b44008aab37f4aceb418a3146b382175, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731712671239 2024-11-15T23:17:55,724 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c82e93185774a5fb9bfae8333bb44af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731712671887 2024-11-15T23:17:55,724 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting cf1e7ae212ea4f459932272d96d3f18f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731712674055 2024-11-15T23:17:55,740 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:55,746 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06815b48d79c548c6fc12bf365c2f5fd#B#compaction#495 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:55,747 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/02c2c22a928946c6ac5e6a4a2ffa43bc is 50, key is test_row_0/B:col10/1731712674063/Put/seqid=0 2024-11-15T23:17:55,748 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111512fea97f5ac747c29d4f09c60b27c459_06815b48d79c548c6fc12bf365c2f5fd store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:55,750 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111512fea97f5ac747c29d4f09c60b27c459_06815b48d79c548c6fc12bf365c2f5fd, store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:55,750 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111512fea97f5ac747c29d4f09c60b27c459_06815b48d79c548c6fc12bf365c2f5fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:55,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742406_1582 (size=12527) 2024-11-15T23:17:55,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742407_1583 (size=4469) 2024-11-15T23:17:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:55,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06815b48d79c548c6fc12bf365c2f5fd 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T23:17:55,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=A 2024-11-15T23:17:55,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:55,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=B 2024-11-15T23:17:55,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:55,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=C 2024-11-15T23:17:55,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:55,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115be7730d9b0ce4a5895b7f1ea5ee07759_06815b48d79c548c6fc12bf365c2f5fd is 50, key is test_row_0/A:col10/1731712675228/Put/seqid=0 2024-11-15T23:17:55,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742408_1584 (size=14794) 2024-11-15T23:17:55,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712735875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712735875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712735879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712735882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712735884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T23:17:55,986 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:17:55,987 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:17:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-15T23:17:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T23:17:55,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712735986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,992 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:17:55,993 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:17:55,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:17:55,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712735995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712735995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712735995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712735995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T23:17:56,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T23:17:56,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:56,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:56,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:56,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:56,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:56,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:56,169 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/02c2c22a928946c6ac5e6a4a2ffa43bc as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/02c2c22a928946c6ac5e6a4a2ffa43bc 2024-11-15T23:17:56,174 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 06815b48d79c548c6fc12bf365c2f5fd/B of 06815b48d79c548c6fc12bf365c2f5fd into 02c2c22a928946c6ac5e6a4a2ffa43bc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:56,174 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:56,174 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., storeName=06815b48d79c548c6fc12bf365c2f5fd/B, priority=12, startTime=1731712675720; duration=0sec 2024-11-15T23:17:56,174 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:17:56,174 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06815b48d79c548c6fc12bf365c2f5fd:B 2024-11-15T23:17:56,176 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:17:56,177 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:17:56,177 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 06815b48d79c548c6fc12bf365c2f5fd/C is initiating minor compaction (all files) 2024-11-15T23:17:56,177 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06815b48d79c548c6fc12bf365c2f5fd/C in TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:56,177 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/ac9caa584ceb4f60b9f39a2904d62ab0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/3110e18872ce4a2080c46c78844b0a48, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/9241baf91f564beca893158bc4a2b98d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/ba9ddff0cd6d4a4db68ccd799d16f7a2] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp, totalSize=47.4 K 2024-11-15T23:17:56,178 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting ac9caa584ceb4f60b9f39a2904d62ab0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731712670083 2024-11-15T23:17:56,178 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06815b48d79c548c6fc12bf365c2f5fd#A#compaction#494 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:56,178 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/670e509773714fb29f4d92cb5dd50abe is 175, key is test_row_0/A:col10/1731712674063/Put/seqid=0 2024-11-15T23:17:56,179 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 3110e18872ce4a2080c46c78844b0a48, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731712671239 2024-11-15T23:17:56,179 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 9241baf91f564beca893158bc4a2b98d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731712671887 2024-11-15T23:17:56,180 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting ba9ddff0cd6d4a4db68ccd799d16f7a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731712674055 2024-11-15T23:17:56,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742409_1585 (size=31481) 2024-11-15T23:17:56,187 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/670e509773714fb29f4d92cb5dd50abe as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/670e509773714fb29f4d92cb5dd50abe 2024-11-15T23:17:56,190 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06815b48d79c548c6fc12bf365c2f5fd#C#compaction#497 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:17:56,190 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/d63a3a0a51e3452ebec3e004635f74af is 50, key is test_row_0/C:col10/1731712674063/Put/seqid=0 2024-11-15T23:17:56,193 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 06815b48d79c548c6fc12bf365c2f5fd/A of 06815b48d79c548c6fc12bf365c2f5fd into 670e509773714fb29f4d92cb5dd50abe(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:56,193 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:56,193 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., storeName=06815b48d79c548c6fc12bf365c2f5fd/A, priority=12, startTime=1731712675719; duration=0sec 2024-11-15T23:17:56,193 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:56,193 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06815b48d79c548c6fc12bf365c2f5fd:A 2024-11-15T23:17:56,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:56,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712736195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:56,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742410_1586 (size=12527) 2024-11-15T23:17:56,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712736205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:56,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:56,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712736206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:56,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712736206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:56,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712736206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:56,274 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:56,276 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115be7730d9b0ce4a5895b7f1ea5ee07759_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115be7730d9b0ce4a5895b7f1ea5ee07759_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:56,277 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/69f4018db5f3401699474ba8cd85a379, store: [table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:56,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/69f4018db5f3401699474ba8cd85a379 is 175, key is test_row_0/A:col10/1731712675228/Put/seqid=0 2024-11-15T23:17:56,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742411_1587 (size=39749) 2024-11-15T23:17:56,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T23:17:56,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:56,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:56,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:56,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:56,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T23:17:56,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T23:17:56,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:56,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:56,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:56,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:56,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:56,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712736505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:56,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:56,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712736515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:56,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:56,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712736515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:56,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712736515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712736515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:56,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T23:17:56,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:56,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:56,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:56,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:56,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:56,611 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/d63a3a0a51e3452ebec3e004635f74af as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/d63a3a0a51e3452ebec3e004635f74af 2024-11-15T23:17:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T23:17:56,616 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 06815b48d79c548c6fc12bf365c2f5fd/C of 06815b48d79c548c6fc12bf365c2f5fd into d63a3a0a51e3452ebec3e004635f74af(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:17:56,616 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:56,616 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., storeName=06815b48d79c548c6fc12bf365c2f5fd/C, priority=12, startTime=1731712675721; duration=0sec 2024-11-15T23:17:56,616 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:17:56,616 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06815b48d79c548c6fc12bf365c2f5fd:C 2024-11-15T23:17:56,681 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/69f4018db5f3401699474ba8cd85a379 2024-11-15T23:17:56,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/0fe9cb74d9974134b1e2a7654fcb0618 is 50, key is test_row_0/B:col10/1731712675228/Put/seqid=0 2024-11-15T23:17:56,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742412_1588 (size=12151) 2024-11-15T23:17:56,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T23:17:56,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:56,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:56,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:56,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:56,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:56,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T23:17:56,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:56,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:56,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:56,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:56,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:56,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:57,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712737015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:57,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712737026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:57,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712737026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:57,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712737026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:57,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712737026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:57,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:57,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:57,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:57,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T23:17:57,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:57,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:57,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:57,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:57,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:17:57,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/0fe9cb74d9974134b1e2a7654fcb0618 2024-11-15T23:17:57,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/6914ffa8906a4346ac3545cafdda6202 is 50, key is test_row_0/C:col10/1731712675228/Put/seqid=0 2024-11-15T23:17:57,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742413_1589 (size=12151) 2024-11-15T23:17:57,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/6914ffa8906a4346ac3545cafdda6202 2024-11-15T23:17:57,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/69f4018db5f3401699474ba8cd85a379 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/69f4018db5f3401699474ba8cd85a379 2024-11-15T23:17:57,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/69f4018db5f3401699474ba8cd85a379, entries=200, sequenceid=173, filesize=38.8 K 2024-11-15T23:17:57,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/0fe9cb74d9974134b1e2a7654fcb0618 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/0fe9cb74d9974134b1e2a7654fcb0618 2024-11-15T23:17:57,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/0fe9cb74d9974134b1e2a7654fcb0618, entries=150, sequenceid=173, filesize=11.9 K 2024-11-15T23:17:57,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/6914ffa8906a4346ac3545cafdda6202 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/6914ffa8906a4346ac3545cafdda6202 2024-11-15T23:17:57,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/6914ffa8906a4346ac3545cafdda6202, entries=150, sequenceid=173, filesize=11.9 K 2024-11-15T23:17:57,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 06815b48d79c548c6fc12bf365c2f5fd in 1255ms, sequenceid=173, compaction requested=false 2024-11-15T23:17:57,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T23:17:57,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T23:17:57,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:57,217 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 06815b48d79c548c6fc12bf365c2f5fd 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T23:17:57,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=A 2024-11-15T23:17:57,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:57,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=B 2024-11-15T23:17:57,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:57,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=C 2024-11-15T23:17:57,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:57,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115302b974fdf71482690dec2e41c6a65ad_06815b48d79c548c6fc12bf365c2f5fd is 50, key is test_row_0/A:col10/1731712675877/Put/seqid=0 2024-11-15T23:17:57,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742414_1590 (size=12304) 2024-11-15T23:17:57,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:57,634 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115302b974fdf71482690dec2e41c6a65ad_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115302b974fdf71482690dec2e41c6a65ad_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:57,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/12aa7ac787e94e0a9e2fab18d73a91cc, store: [table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:57,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/12aa7ac787e94e0a9e2fab18d73a91cc is 175, key is test_row_0/A:col10/1731712675877/Put/seqid=0 2024-11-15T23:17:57,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742415_1591 (size=31105) 2024-11-15T23:17:58,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:17:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:58,056 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/12aa7ac787e94e0a9e2fab18d73a91cc 2024-11-15T23:17:58,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/6645f844ce4b4f9d9092c02847fbba1b is 50, key is test_row_0/B:col10/1731712675877/Put/seqid=0 2024-11-15T23:17:58,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742416_1592 (size=12151) 2024-11-15T23:17:58,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712738078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712738080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712738080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712738084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712738084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T23:17:58,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712738195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712738195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712738195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712738195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712738195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712738405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712738405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712738405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712738406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712738406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,467 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/6645f844ce4b4f9d9092c02847fbba1b 2024-11-15T23:17:58,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/2943638df95841bfb24ca8fc69eced20 is 50, key is test_row_0/C:col10/1731712675877/Put/seqid=0 2024-11-15T23:17:58,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742417_1593 (size=12151) 2024-11-15T23:17:58,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712738715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712738716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712738716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712738716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712738716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:58,878 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/2943638df95841bfb24ca8fc69eced20 2024-11-15T23:17:58,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/12aa7ac787e94e0a9e2fab18d73a91cc as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/12aa7ac787e94e0a9e2fab18d73a91cc 2024-11-15T23:17:58,895 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/12aa7ac787e94e0a9e2fab18d73a91cc, entries=150, sequenceid=192, filesize=30.4 K 2024-11-15T23:17:58,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/6645f844ce4b4f9d9092c02847fbba1b as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/6645f844ce4b4f9d9092c02847fbba1b 2024-11-15T23:17:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,899 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/6645f844ce4b4f9d9092c02847fbba1b, entries=150, sequenceid=192, filesize=11.9 K 2024-11-15T23:17:58,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/2943638df95841bfb24ca8fc69eced20 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/2943638df95841bfb24ca8fc69eced20 2024-11-15T23:17:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,904 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/2943638df95841bfb24ca8fc69eced20, entries=150, sequenceid=192, filesize=11.9 K 2024-11-15T23:17:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,905 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 06815b48d79c548c6fc12bf365c2f5fd in 1688ms, sequenceid=192, compaction requested=true 2024-11-15T23:17:58,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:17:58,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:17:58,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-15T23:17:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-15T23:17:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-15T23:17:58,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9140 sec 2024-11-15T23:17:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.9220 sec 2024-11-15T23:17:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:59,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06815b48d79c548c6fc12bf365c2f5fd 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T23:17:59,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=A 2024-11-15T23:17:59,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:59,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=B 2024-11-15T23:17:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:59,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=C 2024-11-15T23:17:59,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:17:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115168ed4e55cfe474e8c6882e5629703f5_06815b48d79c548c6fc12bf365c2f5fd is 50, key is test_row_0/A:col10/1731712678077/Put/seqid=0 2024-11-15T23:17:59,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742418_1594 (size=12304) 2024-11-15T23:17:59,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712739292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712739293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712739293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712739294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:59,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:59,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712739301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:59,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712739405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712739405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712739405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:59,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712739406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:59,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:59,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712739415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:59,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712739615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712739615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:59,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712739615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:59,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712739616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:59,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712739625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:59,661 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:17:59,664 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115168ed4e55cfe474e8c6882e5629703f5_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115168ed4e55cfe474e8c6882e5629703f5_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:17:59,665 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/97b0c5532ac14696a4bf4c24330d44ee, store: [table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:17:59,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/97b0c5532ac14696a4bf4c24330d44ee is 175, key is test_row_0/A:col10/1731712678077/Put/seqid=0 2024-11-15T23:17:59,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742419_1595 (size=31101) 2024-11-15T23:17:59,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712739925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712739925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:59,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712739925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712739925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:17:59,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:17:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712739935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:17:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:17:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:00,068 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/97b0c5532ac14696a4bf4c24330d44ee 2024-11-15T23:18:00,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/a7fc6d0bc618443bb1af9e56ce4b1a2d is 50, key is test_row_0/B:col10/1731712678077/Put/seqid=0 2024-11-15T23:18:00,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742420_1596 (size=9757) 2024-11-15T23:18:00,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T23:18:00,166 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:18:00,166 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:18:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-15T23:18:00,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T23:18:00,168 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:18:00,169 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:18:00,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:18:00,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T23:18:00,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T23:18:00,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:00,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:18:00,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:00,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:00,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:00,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712740436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:00,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:00,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:00,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:00,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712740436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:00,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:00,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712740436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712740436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:00,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712740445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:00,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T23:18:00,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:00,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:18:00,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:00,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:00,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T23:18:00,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/a7fc6d0bc618443bb1af9e56ce4b1a2d 2024-11-15T23:18:00,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/64735b1959f74fc5ab58e1f2338ae8dc is 50, key is test_row_0/C:col10/1731712678077/Put/seqid=0 2024-11-15T23:18:00,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742421_1597 (size=9757) 2024-11-15T23:18:00,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T23:18:00,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:00,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:18:00,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:00,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:00,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:00,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T23:18:00,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:00,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:18:00,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:00,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:00,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T23:18:00,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T23:18:00,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:00,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:18:00,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:00,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:00,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:00,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/64735b1959f74fc5ab58e1f2338ae8dc 2024-11-15T23:18:00,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/97b0c5532ac14696a4bf4c24330d44ee as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/97b0c5532ac14696a4bf4c24330d44ee 2024-11-15T23:18:00,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/97b0c5532ac14696a4bf4c24330d44ee, entries=150, sequenceid=210, filesize=30.4 K 2024-11-15T23:18:00,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/a7fc6d0bc618443bb1af9e56ce4b1a2d as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/a7fc6d0bc618443bb1af9e56ce4b1a2d 2024-11-15T23:18:00,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/a7fc6d0bc618443bb1af9e56ce4b1a2d, entries=100, sequenceid=210, filesize=9.5 K 2024-11-15T23:18:00,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/64735b1959f74fc5ab58e1f2338ae8dc as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/64735b1959f74fc5ab58e1f2338ae8dc 2024-11-15T23:18:00,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/64735b1959f74fc5ab58e1f2338ae8dc, entries=100, sequenceid=210, filesize=9.5 K 2024-11-15T23:18:00,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 06815b48d79c548c6fc12bf365c2f5fd in 1717ms, sequenceid=210, compaction requested=true 2024-11-15T23:18:00,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:18:00,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06815b48d79c548c6fc12bf365c2f5fd:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:18:00,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:18:00,949 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:18:00,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06815b48d79c548c6fc12bf365c2f5fd:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:18:00,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:18:00,949 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:18:00,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06815b48d79c548c6fc12bf365c2f5fd:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:18:00,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:18:00,950 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46586 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:18:00,950 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133436 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:18:00,950 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 06815b48d79c548c6fc12bf365c2f5fd/A is initiating minor compaction (all files) 2024-11-15T23:18:00,950 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 06815b48d79c548c6fc12bf365c2f5fd/B is initiating minor compaction (all files) 2024-11-15T23:18:00,950 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06815b48d79c548c6fc12bf365c2f5fd/B in TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:00,950 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06815b48d79c548c6fc12bf365c2f5fd/A in TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:00,950 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/670e509773714fb29f4d92cb5dd50abe, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/69f4018db5f3401699474ba8cd85a379, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/12aa7ac787e94e0a9e2fab18d73a91cc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/97b0c5532ac14696a4bf4c24330d44ee] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp, totalSize=130.3 K 2024-11-15T23:18:00,950 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/02c2c22a928946c6ac5e6a4a2ffa43bc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/0fe9cb74d9974134b1e2a7654fcb0618, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/6645f844ce4b4f9d9092c02847fbba1b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/a7fc6d0bc618443bb1af9e56ce4b1a2d] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp, totalSize=45.5 K 2024-11-15T23:18:00,950 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:00,950 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. files: [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/670e509773714fb29f4d92cb5dd50abe, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/69f4018db5f3401699474ba8cd85a379, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/12aa7ac787e94e0a9e2fab18d73a91cc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/97b0c5532ac14696a4bf4c24330d44ee] 2024-11-15T23:18:00,950 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 02c2c22a928946c6ac5e6a4a2ffa43bc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731712674055 2024-11-15T23:18:00,950 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fe9cb74d9974134b1e2a7654fcb0618, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731712675222 2024-11-15T23:18:00,950 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 670e509773714fb29f4d92cb5dd50abe, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731712674055 2024-11-15T23:18:00,951 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 6645f844ce4b4f9d9092c02847fbba1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731712675873 2024-11-15T23:18:00,951 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69f4018db5f3401699474ba8cd85a379, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731712675222 2024-11-15T23:18:00,951 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting a7fc6d0bc618443bb1af9e56ce4b1a2d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731712678077 2024-11-15T23:18:00,951 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12aa7ac787e94e0a9e2fab18d73a91cc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731712675873 2024-11-15T23:18:00,952 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97b0c5532ac14696a4bf4c24330d44ee, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731712678077 2024-11-15T23:18:00,959 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06815b48d79c548c6fc12bf365c2f5fd#B#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:18:00,960 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/1c49fd5a83704af28ab7aae24b29151a is 50, key is test_row_0/B:col10/1731712678077/Put/seqid=0 2024-11-15T23:18:00,961 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:18:00,968 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115bbd083bd6d4b431289d08038e127ad6c_06815b48d79c548c6fc12bf365c2f5fd store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:18:00,970 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115bbd083bd6d4b431289d08038e127ad6c_06815b48d79c548c6fc12bf365c2f5fd, store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:18:00,971 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bbd083bd6d4b431289d08038e127ad6c_06815b48d79c548c6fc12bf365c2f5fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:18:00,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742422_1598 (size=12663) 2024-11-15T23:18:00,977 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/1c49fd5a83704af28ab7aae24b29151a as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/1c49fd5a83704af28ab7aae24b29151a 2024-11-15T23:18:00,980 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 06815b48d79c548c6fc12bf365c2f5fd/B of 06815b48d79c548c6fc12bf365c2f5fd into 1c49fd5a83704af28ab7aae24b29151a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:18:00,980 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:18:00,980 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., storeName=06815b48d79c548c6fc12bf365c2f5fd/B, priority=12, startTime=1731712680949; duration=0sec 2024-11-15T23:18:00,980 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:18:00,980 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06815b48d79c548c6fc12bf365c2f5fd:B 2024-11-15T23:18:00,980 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:18:00,981 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46586 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:18:00,981 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 06815b48d79c548c6fc12bf365c2f5fd/C is initiating minor compaction (all files) 2024-11-15T23:18:00,981 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06815b48d79c548c6fc12bf365c2f5fd/C in TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:00,981 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/d63a3a0a51e3452ebec3e004635f74af, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/6914ffa8906a4346ac3545cafdda6202, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/2943638df95841bfb24ca8fc69eced20, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/64735b1959f74fc5ab58e1f2338ae8dc] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp, totalSize=45.5 K 2024-11-15T23:18:00,982 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting d63a3a0a51e3452ebec3e004635f74af, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731712674055 2024-11-15T23:18:00,982 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 6914ffa8906a4346ac3545cafdda6202, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731712675222 2024-11-15T23:18:00,982 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 2943638df95841bfb24ca8fc69eced20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731712675873 2024-11-15T23:18:00,982 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 64735b1959f74fc5ab58e1f2338ae8dc, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731712678077 2024-11-15T23:18:00,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742423_1599 (size=4469) 2024-11-15T23:18:00,987 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06815b48d79c548c6fc12bf365c2f5fd#A#compaction#507 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:18:00,987 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/5080d1eb3d33416eb9705e00808dbec9 is 175, key is test_row_0/A:col10/1731712678077/Put/seqid=0 2024-11-15T23:18:01,000 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06815b48d79c548c6fc12bf365c2f5fd#C#compaction#508 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:18:01,001 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/bea4cd672dc9437fb72a5f3a7c68b706 is 50, key is test_row_0/C:col10/1731712678077/Put/seqid=0 2024-11-15T23:18:01,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742424_1600 (size=31724) 2024-11-15T23:18:01,007 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/5080d1eb3d33416eb9705e00808dbec9 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/5080d1eb3d33416eb9705e00808dbec9 2024-11-15T23:18:01,011 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 06815b48d79c548c6fc12bf365c2f5fd/A of 06815b48d79c548c6fc12bf365c2f5fd into 5080d1eb3d33416eb9705e00808dbec9(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:18:01,011 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:18:01,012 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., storeName=06815b48d79c548c6fc12bf365c2f5fd/A, priority=12, startTime=1731712680949; duration=0sec 2024-11-15T23:18:01,012 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:18:01,012 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06815b48d79c548c6fc12bf365c2f5fd:A 2024-11-15T23:18:01,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742425_1601 (size=12663) 2024-11-15T23:18:01,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T23:18:01,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:01,083 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 06815b48d79c548c6fc12bf365c2f5fd 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T23:18:01,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=A 2024-11-15T23:18:01,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:18:01,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=B 2024-11-15T23:18:01,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:18:01,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=C 2024-11-15T23:18:01,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:18:01,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111531aca22e94a74f308f524d80e91240ea_06815b48d79c548c6fc12bf365c2f5fd is 50, key is test_row_0/A:col10/1731712679251/Put/seqid=0 2024-11-15T23:18:01,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742426_1602 (size=12304) 2024-11-15T23:18:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T23:18:01,417 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/bea4cd672dc9437fb72a5f3a7c68b706 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/bea4cd672dc9437fb72a5f3a7c68b706 2024-11-15T23:18:01,420 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 06815b48d79c548c6fc12bf365c2f5fd/C of 06815b48d79c548c6fc12bf365c2f5fd into bea4cd672dc9437fb72a5f3a7c68b706(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:18:01,420 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:18:01,420 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., storeName=06815b48d79c548c6fc12bf365c2f5fd/C, priority=12, startTime=1731712680949; duration=0sec 2024-11-15T23:18:01,420 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:18:01,420 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06815b48d79c548c6fc12bf365c2f5fd:C 2024-11-15T23:18:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:01,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:18:01,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712741461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:01,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712741461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:01,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712741462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:01,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712741463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:01,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712741465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:01,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:18:01,500 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111531aca22e94a74f308f524d80e91240ea_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111531aca22e94a74f308f524d80e91240ea_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:01,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/29d602fd44bf41018f4276c64ccc8caf, store: [table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:18:01,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/29d602fd44bf41018f4276c64ccc8caf is 175, key is test_row_0/A:col10/1731712679251/Put/seqid=0 2024-11-15T23:18:01,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742427_1603 (size=31105) 2024-11-15T23:18:01,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712741575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:01,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712741575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:01,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712741575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712741575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712741575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:01,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:01,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712741785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:01,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712741785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712741785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712741785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:01,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712741785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:01,906 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/29d602fd44bf41018f4276c64ccc8caf 2024-11-15T23:18:01,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/b38fb512553d4f40b3a6686ab867e569 is 50, key is test_row_0/B:col10/1731712679251/Put/seqid=0 2024-11-15T23:18:01,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742428_1604 (size=12151) 2024-11-15T23:18:01,926 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/b38fb512553d4f40b3a6686ab867e569 2024-11-15T23:18:01,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/e5f0ccb7650e4a22b04eb6e7e3185f77 is 50, key is test_row_0/C:col10/1731712679251/Put/seqid=0 2024-11-15T23:18:01,968 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T23:18:01,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742429_1605 (size=12151) 2024-11-15T23:18:01,987 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/e5f0ccb7650e4a22b04eb6e7e3185f77 2024-11-15T23:18:01,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/29d602fd44bf41018f4276c64ccc8caf as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/29d602fd44bf41018f4276c64ccc8caf 2024-11-15T23:18:01,996 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/29d602fd44bf41018f4276c64ccc8caf, entries=150, sequenceid=226, filesize=30.4 K 2024-11-15T23:18:01,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/b38fb512553d4f40b3a6686ab867e569 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/b38fb512553d4f40b3a6686ab867e569 2024-11-15T23:18:02,009 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/b38fb512553d4f40b3a6686ab867e569, entries=150, sequenceid=226, filesize=11.9 K 2024-11-15T23:18:02,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/e5f0ccb7650e4a22b04eb6e7e3185f77 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/e5f0ccb7650e4a22b04eb6e7e3185f77 2024-11-15T23:18:02,020 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/e5f0ccb7650e4a22b04eb6e7e3185f77, entries=150, sequenceid=226, filesize=11.9 K 2024-11-15T23:18:02,022 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 06815b48d79c548c6fc12bf365c2f5fd in 938ms, sequenceid=226, compaction requested=false 2024-11-15T23:18:02,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:18:02,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:02,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-15T23:18:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-15T23:18:02,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-15T23:18:02,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8550 sec 2024-11-15T23:18:02,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.8640 sec 2024-11-15T23:18:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:02,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06815b48d79c548c6fc12bf365c2f5fd 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T23:18:02,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=A 2024-11-15T23:18:02,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:18:02,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=B 2024-11-15T23:18:02,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:18:02,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=C 2024-11-15T23:18:02,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:18:02,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115945b82c3e37d4715851daf75064779b9_06815b48d79c548c6fc12bf365c2f5fd is 50, key is test_row_0/A:col10/1731712682096/Put/seqid=0 2024-11-15T23:18:02,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742430_1606 (size=12304) 2024-11-15T23:18:02,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712742104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712742105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712742106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712742107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712742108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712742215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712742215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712742215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712742216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712742216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T23:18:02,316 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:18:02,316 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T23:18:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-15T23:18:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T23:18:02,317 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:18:02,318 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:18:02,318 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:18:02,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T23:18:02,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712742426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712742426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712742426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712742426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712742426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T23:18:02,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:02,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:18:02,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:02,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:02,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:02,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:02,509 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:18:02,512 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115945b82c3e37d4715851daf75064779b9_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115945b82c3e37d4715851daf75064779b9_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:02,512 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/283f96d53b1e40d3bf62be3261ee7930, store: [table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:18:02,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/283f96d53b1e40d3bf62be3261ee7930 is 175, key is test_row_0/A:col10/1731712682096/Put/seqid=0 2024-11-15T23:18:02,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742431_1607 (size=31105) 2024-11-15T23:18:02,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T23:18:02,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:02,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:18:02,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:02,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:02,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T23:18:02,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712742735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712742735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712742736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712742736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712742736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:02,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T23:18:02,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:02,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:18:02,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:02,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:02,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:02,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:02,921 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/283f96d53b1e40d3bf62be3261ee7930 2024-11-15T23:18:02,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/974fe27af73b47ef89ca454e7c0754b8 is 50, key is test_row_0/B:col10/1731712682096/Put/seqid=0 2024-11-15T23:18:02,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T23:18:02,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:02,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:18:02,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:02,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:02,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:02,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742432_1608 (size=12151) 2024-11-15T23:18:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T23:18:03,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T23:18:03,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:03,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:18:03,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:03,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:03,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:03,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T23:18:03,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:03,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:18:03,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:03,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:03,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:03,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51688 deadline: 1731712743245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:03,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51720 deadline: 1731712743246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:03,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51718 deadline: 1731712743246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:03,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51660 deadline: 1731712743255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:03,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T23:18:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38353 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51710 deadline: 1731712743255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 , the old value is region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:18:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., hostname=d6ecb9b7dda1,38353,1731712503763, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:18:03,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/974fe27af73b47ef89ca454e7c0754b8 2024-11-15T23:18:03,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/6e6ace6e6bef4761975f5115e4d5bb71 is 50, key is test_row_0/C:col10/1731712682096/Put/seqid=0 2024-11-15T23:18:03,343 INFO [Thread-2416 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2416. 2024-11-15T23:18:03,343 DEBUG [Thread-2416 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:18:03,343 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:03,343 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:03,343 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:18:03,344 INFO [Thread-2410 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2410. 2024-11-15T23:18:03,344 DEBUG [Thread-2410 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:18:03,344 DEBUG [Thread-2410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:03,344 DEBUG [Thread-2410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:03,344 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:18:03,345 INFO [Thread-2408 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2408. 2024-11-15T23:18:03,345 DEBUG [Thread-2408 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:18:03,345 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:03,345 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:03,345 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:18:03,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742433_1609 (size=12151) 2024-11-15T23:18:03,347 INFO [Thread-2412 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2412. 2024-11-15T23:18:03,347 INFO [Thread-2414 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2414. 2024-11-15T23:18:03,347 DEBUG [Thread-2412 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:18:03,347 DEBUG [Thread-2414 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:18:03,347 DEBUG [Thread-2412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:03,347 DEBUG [Thread-2414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:03,347 DEBUG [Thread-2412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:03,347 DEBUG [Thread-2414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:03,347 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:18:03,347 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:18:03,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T23:18:03,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:03,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:18:03,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:03,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:03,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T23:18:03,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T23:18:03,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:03,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:18:03,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:03,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:03,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:03,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T23:18:03,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:03,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:18:03,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:03,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:03,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:18:03,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/6e6ace6e6bef4761975f5115e4d5bb71 2024-11-15T23:18:03,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/283f96d53b1e40d3bf62be3261ee7930 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/283f96d53b1e40d3bf62be3261ee7930 2024-11-15T23:18:03,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/283f96d53b1e40d3bf62be3261ee7930, entries=150, sequenceid=247, filesize=30.4 K 2024-11-15T23:18:03,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/974fe27af73b47ef89ca454e7c0754b8 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/974fe27af73b47ef89ca454e7c0754b8 2024-11-15T23:18:03,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/974fe27af73b47ef89ca454e7c0754b8, entries=150, sequenceid=247, filesize=11.9 K 2024-11-15T23:18:03,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/6e6ace6e6bef4761975f5115e4d5bb71 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/6e6ace6e6bef4761975f5115e4d5bb71 2024-11-15T23:18:03,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/6e6ace6e6bef4761975f5115e4d5bb71, entries=150, sequenceid=247, filesize=11.9 K 2024-11-15T23:18:03,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 06815b48d79c548c6fc12bf365c2f5fd in 1661ms, sequenceid=247, compaction requested=true 2024-11-15T23:18:03,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:18:03,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06815b48d79c548c6fc12bf365c2f5fd:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:18:03,759 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:18:03,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:18:03,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06815b48d79c548c6fc12bf365c2f5fd:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:18:03,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:18:03,759 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:18:03,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06815b48d79c548c6fc12bf365c2f5fd:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:18:03,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:18:03,760 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93934 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:18:03,760 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1541): 06815b48d79c548c6fc12bf365c2f5fd/A is initiating minor compaction (all files) 2024-11-15T23:18:03,760 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:18:03,760 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 06815b48d79c548c6fc12bf365c2f5fd/B is initiating minor compaction (all files) 2024-11-15T23:18:03,760 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06815b48d79c548c6fc12bf365c2f5fd/A in TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:03,760 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06815b48d79c548c6fc12bf365c2f5fd/B in TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:03,760 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/1c49fd5a83704af28ab7aae24b29151a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/b38fb512553d4f40b3a6686ab867e569, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/974fe27af73b47ef89ca454e7c0754b8] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp, totalSize=36.1 K 2024-11-15T23:18:03,760 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/5080d1eb3d33416eb9705e00808dbec9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/29d602fd44bf41018f4276c64ccc8caf, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/283f96d53b1e40d3bf62be3261ee7930] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp, totalSize=91.7 K 2024-11-15T23:18:03,760 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:03,760 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. files: [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/5080d1eb3d33416eb9705e00808dbec9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/29d602fd44bf41018f4276c64ccc8caf, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/283f96d53b1e40d3bf62be3261ee7930] 2024-11-15T23:18:03,760 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c49fd5a83704af28ab7aae24b29151a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731712675882 2024-11-15T23:18:03,760 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5080d1eb3d33416eb9705e00808dbec9, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731712675882 2024-11-15T23:18:03,761 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29d602fd44bf41018f4276c64ccc8caf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731712679251 2024-11-15T23:18:03,761 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting b38fb512553d4f40b3a6686ab867e569, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731712679251 2024-11-15T23:18:03,761 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 974fe27af73b47ef89ca454e7c0754b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731712681462 2024-11-15T23:18:03,761 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] compactions.Compactor(225): Compacting 283f96d53b1e40d3bf62be3261ee7930, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731712681462 2024-11-15T23:18:03,772 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06815b48d79c548c6fc12bf365c2f5fd#B#compaction#515 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:18:03,772 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:18:03,773 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/01838482d0214862a037149dfaa11b30 is 50, key is test_row_0/B:col10/1731712682096/Put/seqid=0 2024-11-15T23:18:03,774 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411151e277cb3503948108c185f54c8668c04_06815b48d79c548c6fc12bf365c2f5fd store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:18:03,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742434_1610 (size=12765) 2024-11-15T23:18:03,796 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/01838482d0214862a037149dfaa11b30 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/01838482d0214862a037149dfaa11b30 2024-11-15T23:18:03,800 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06815b48d79c548c6fc12bf365c2f5fd/B of 06815b48d79c548c6fc12bf365c2f5fd into 01838482d0214862a037149dfaa11b30(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:18:03,801 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:18:03,801 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., storeName=06815b48d79c548c6fc12bf365c2f5fd/B, priority=13, startTime=1731712683759; duration=0sec 2024-11-15T23:18:03,801 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:18:03,801 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06815b48d79c548c6fc12bf365c2f5fd:B 2024-11-15T23:18:03,801 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:18:03,801 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411151e277cb3503948108c185f54c8668c04_06815b48d79c548c6fc12bf365c2f5fd, store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:18:03,802 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151e277cb3503948108c185f54c8668c04_06815b48d79c548c6fc12bf365c2f5fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:18:03,802 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:18:03,802 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1541): 06815b48d79c548c6fc12bf365c2f5fd/C is initiating minor compaction (all files) 2024-11-15T23:18:03,802 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06815b48d79c548c6fc12bf365c2f5fd/C in TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:03,802 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/bea4cd672dc9437fb72a5f3a7c68b706, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/e5f0ccb7650e4a22b04eb6e7e3185f77, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/6e6ace6e6bef4761975f5115e4d5bb71] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp, totalSize=36.1 K 2024-11-15T23:18:03,802 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting bea4cd672dc9437fb72a5f3a7c68b706, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731712675882 2024-11-15T23:18:03,802 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting e5f0ccb7650e4a22b04eb6e7e3185f77, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731712679251 2024-11-15T23:18:03,803 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e6ace6e6bef4761975f5115e4d5bb71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731712681462 2024-11-15T23:18:03,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742435_1611 (size=4469) 2024-11-15T23:18:03,815 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06815b48d79c548c6fc12bf365c2f5fd#A#compaction#516 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:18:03,816 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/216ffe2f38b34610ad8880f9e186062c is 175, key is test_row_0/A:col10/1731712682096/Put/seqid=0 2024-11-15T23:18:03,816 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06815b48d79c548c6fc12bf365c2f5fd#C#compaction#517 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:18:03,816 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/9c93c9edf27f4a32b7a0d2a2ade09d09 is 50, key is test_row_0/C:col10/1731712682096/Put/seqid=0 2024-11-15T23:18:03,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742436_1612 (size=31719) 2024-11-15T23:18:03,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742437_1613 (size=12765) 2024-11-15T23:18:03,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38353 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T23:18:03,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:03,843 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 06815b48d79c548c6fc12bf365c2f5fd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T23:18:03,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=A 2024-11-15T23:18:03,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:18:03,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=B 2024-11-15T23:18:03,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:18:03,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=C 2024-11-15T23:18:03,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:18:03,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115001adfe8324c42f78b1982c87a0f7cfb_06815b48d79c548c6fc12bf365c2f5fd is 50, key is test_row_0/A:col10/1731712682105/Put/seqid=0 2024-11-15T23:18:03,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742438_1614 (size=12354) 2024-11-15T23:18:04,225 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/216ffe2f38b34610ad8880f9e186062c as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/216ffe2f38b34610ad8880f9e186062c 2024-11-15T23:18:04,229 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06815b48d79c548c6fc12bf365c2f5fd/A of 06815b48d79c548c6fc12bf365c2f5fd into 216ffe2f38b34610ad8880f9e186062c(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:18:04,229 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:18:04,229 INFO [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., storeName=06815b48d79c548c6fc12bf365c2f5fd/A, priority=13, startTime=1731712683759; duration=0sec 2024-11-15T23:18:04,229 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:18:04,229 DEBUG [RS:0;d6ecb9b7dda1:38353-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06815b48d79c548c6fc12bf365c2f5fd:A 2024-11-15T23:18:04,230 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/9c93c9edf27f4a32b7a0d2a2ade09d09 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/9c93c9edf27f4a32b7a0d2a2ade09d09 2024-11-15T23:18:04,233 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06815b48d79c548c6fc12bf365c2f5fd/C of 06815b48d79c548c6fc12bf365c2f5fd into 9c93c9edf27f4a32b7a0d2a2ade09d09(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:18:04,233 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:18:04,233 INFO [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., storeName=06815b48d79c548c6fc12bf365c2f5fd/C, priority=13, startTime=1731712683759; duration=0sec 2024-11-15T23:18:04,233 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:18:04,233 DEBUG [RS:0;d6ecb9b7dda1:38353-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06815b48d79c548c6fc12bf365c2f5fd:C 2024-11-15T23:18:04,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:18:04,256 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115001adfe8324c42f78b1982c87a0f7cfb_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115001adfe8324c42f78b1982c87a0f7cfb_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:04,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/343494bb8e3a44618ffef0dc41f795c5, store: [table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:18:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38353 {}] regionserver.HRegion(8855): Flush requested on 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:04,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. as already flushing 2024-11-15T23:18:04,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/343494bb8e3a44618ffef0dc41f795c5 is 175, key is test_row_0/A:col10/1731712682105/Put/seqid=0 2024-11-15T23:18:04,257 INFO [Thread-2403 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2403. 2024-11-15T23:18:04,257 INFO [Thread-2401 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2401. 2024-11-15T23:18:04,257 DEBUG [Thread-2403 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:18:04,257 DEBUG [Thread-2403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:04,257 DEBUG [Thread-2401 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:18:04,257 DEBUG [Thread-2401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:04,257 DEBUG [Thread-2403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:04,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:18:04,257 DEBUG [Thread-2401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:04,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:18:04,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742439_1615 (size=31155) 2024-11-15T23:18:04,267 INFO [Thread-2399 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2399. 2024-11-15T23:18:04,267 INFO [Thread-2405 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2405. 2024-11-15T23:18:04,267 INFO [Thread-2397 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2397. 2024-11-15T23:18:04,267 DEBUG [Thread-2399 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:18:04,267 DEBUG [Thread-2405 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:18:04,267 DEBUG [Thread-2397 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:18:04,267 DEBUG [Thread-2399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:04,267 DEBUG [Thread-2405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:04,267 DEBUG [Thread-2397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:04,267 DEBUG [Thread-2399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:04,267 DEBUG [Thread-2405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:04,267 DEBUG [Thread-2397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:04,267 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:18:04,267 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:18:04,267 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:18:04,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T23:18:04,660 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/343494bb8e3a44618ffef0dc41f795c5 2024-11-15T23:18:04,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/e27c7d1790f944cfa629a93477c9672b is 50, key is test_row_0/B:col10/1731712682105/Put/seqid=0 2024-11-15T23:18:04,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742440_1616 (size=12201) 2024-11-15T23:18:05,068 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/e27c7d1790f944cfa629a93477c9672b 2024-11-15T23:18:05,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/a0535d8a51974d17ad22a497f7f52262 is 50, key is test_row_0/C:col10/1731712682105/Put/seqid=0 2024-11-15T23:18:05,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742441_1617 (size=12201) 2024-11-15T23:18:05,477 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/a0535d8a51974d17ad22a497f7f52262 2024-11-15T23:18:05,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/343494bb8e3a44618ffef0dc41f795c5 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/343494bb8e3a44618ffef0dc41f795c5 2024-11-15T23:18:05,483 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/343494bb8e3a44618ffef0dc41f795c5, entries=150, sequenceid=260, filesize=30.4 K 2024-11-15T23:18:05,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/e27c7d1790f944cfa629a93477c9672b as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/e27c7d1790f944cfa629a93477c9672b 2024-11-15T23:18:05,487 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/e27c7d1790f944cfa629a93477c9672b, entries=150, sequenceid=260, filesize=11.9 K 2024-11-15T23:18:05,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/a0535d8a51974d17ad22a497f7f52262 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/a0535d8a51974d17ad22a497f7f52262 2024-11-15T23:18:05,489 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/a0535d8a51974d17ad22a497f7f52262, entries=150, sequenceid=260, filesize=11.9 K 2024-11-15T23:18:05,490 INFO [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=33.54 KB/34350 for 06815b48d79c548c6fc12bf365c2f5fd in 1647ms, sequenceid=260, compaction requested=false 2024-11-15T23:18:05,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 06815b48d79c548c6fc12bf365c2f5fd: 2024-11-15T23:18:05,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:05,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6ecb9b7dda1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-15T23:18:05,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-15T23:18:05,492 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-15T23:18:05,492 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1730 sec 2024-11-15T23:18:05,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 3.1760 sec 2024-11-15T23:18:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T23:18:06,486 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:18:06,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T23:18:06,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-15T23:18:06,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-15T23:18:06,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-15T23:18:06,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-15T23:18:06,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-15T23:18:06,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T23:18:06,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3444 2024-11-15T23:18:06,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3349 2024-11-15T23:18:06,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3335 2024-11-15T23:18:06,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3442 2024-11-15T23:18:06,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3347 2024-11-15T23:18:06,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T23:18:06,486 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T23:18:06,486 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T23:18:06,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:06,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:06,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:18:06,487 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T23:18:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T23:18:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T23:18:06,490 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712686489"}]},"ts":"1731712686489"} 2024-11-15T23:18:06,491 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T23:18:06,491 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T23:18:06,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T23:18:06,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=06815b48d79c548c6fc12bf365c2f5fd, UNASSIGN}] 2024-11-15T23:18:06,493 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=06815b48d79c548c6fc12bf365c2f5fd, UNASSIGN 2024-11-15T23:18:06,494 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=06815b48d79c548c6fc12bf365c2f5fd, regionState=CLOSING, regionLocation=d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:06,494 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43421 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=d6ecb9b7dda1,38353,1731712503763, table=TestAcidGuarantees, region=06815b48d79c548c6fc12bf365c2f5fd. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T23:18:06,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=06815b48d79c548c6fc12bf365c2f5fd, UNASSIGN because future has completed 2024-11-15T23:18:06,495 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T23:18:06,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763}] 2024-11-15T23:18:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T23:18:06,647 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:06,647 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T23:18:06,647 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing 06815b48d79c548c6fc12bf365c2f5fd, disabling compactions & flushes 2024-11-15T23:18:06,647 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:06,647 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:06,647 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. after waiting 0 ms 2024-11-15T23:18:06,647 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:06,647 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing 06815b48d79c548c6fc12bf365c2f5fd 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-15T23:18:06,647 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=A 2024-11-15T23:18:06,648 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:18:06,648 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=B 2024-11-15T23:18:06,648 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:18:06,648 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06815b48d79c548c6fc12bf365c2f5fd, store=C 2024-11-15T23:18:06,648 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T23:18:06,652 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c194e18cb1d74cd0be086fa8b3cc0d15_06815b48d79c548c6fc12bf365c2f5fd is 50, key is test_row_0/A:col10/1731712684265/Put/seqid=0 2024-11-15T23:18:06,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742442_1618 (size=9914) 2024-11-15T23:18:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T23:18:07,055 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:18:07,058 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c194e18cb1d74cd0be086fa8b3cc0d15_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c194e18cb1d74cd0be086fa8b3cc0d15_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:07,058 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/07fc4d5b96fc44c984dd82ac75503124, store: [table=TestAcidGuarantees family=A region=06815b48d79c548c6fc12bf365c2f5fd] 2024-11-15T23:18:07,059 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/07fc4d5b96fc44c984dd82ac75503124 is 175, key is test_row_0/A:col10/1731712684265/Put/seqid=0 2024-11-15T23:18:07,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742443_1619 (size=22561) 2024-11-15T23:18:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T23:18:07,462 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/07fc4d5b96fc44c984dd82ac75503124 2024-11-15T23:18:07,467 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/8a4279e18e6b441e87ac8b102b9bcbd2 is 50, key is test_row_0/B:col10/1731712684265/Put/seqid=0 2024-11-15T23:18:07,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742444_1620 (size=9857) 2024-11-15T23:18:07,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T23:18:07,870 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/8a4279e18e6b441e87ac8b102b9bcbd2 2024-11-15T23:18:07,876 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/8753e00c6dde4b608a1e73f768e8b4f8 is 50, key is test_row_0/C:col10/1731712684265/Put/seqid=0 2024-11-15T23:18:07,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742445_1621 (size=9857) 2024-11-15T23:18:08,279 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/8753e00c6dde4b608a1e73f768e8b4f8 2024-11-15T23:18:08,282 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/A/07fc4d5b96fc44c984dd82ac75503124 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/07fc4d5b96fc44c984dd82ac75503124 2024-11-15T23:18:08,285 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/07fc4d5b96fc44c984dd82ac75503124, entries=100, sequenceid=270, filesize=22.0 K 2024-11-15T23:18:08,286 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/B/8a4279e18e6b441e87ac8b102b9bcbd2 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/8a4279e18e6b441e87ac8b102b9bcbd2 2024-11-15T23:18:08,288 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/8a4279e18e6b441e87ac8b102b9bcbd2, entries=100, sequenceid=270, filesize=9.6 K 2024-11-15T23:18:08,289 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/.tmp/C/8753e00c6dde4b608a1e73f768e8b4f8 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/8753e00c6dde4b608a1e73f768e8b4f8 2024-11-15T23:18:08,291 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/8753e00c6dde4b608a1e73f768e8b4f8, entries=100, sequenceid=270, filesize=9.6 K 2024-11-15T23:18:08,292 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 06815b48d79c548c6fc12bf365c2f5fd in 1645ms, sequenceid=270, compaction requested=true 2024-11-15T23:18:08,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/863dcdda3ffe40198d91c3f7a06fa9a0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/c6d67fbe7a994cc9bf5a3de67482ada0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/ee090792f9794d97a76ed32fe4ac71e2, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/376af1ed0cff4b188f158768566ee302, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/ab983dae524d400d8afabb9cbc4e4d69, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/2ddaed68bc654196a31408ba5f004707, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/e101d612c8a243bc9f1025fbd0dc3278, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/f32e8e274cde427689d9c5492b0ac52d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/f1a5c7dd95fb49d1ac9bcba6f456da34, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/9c2d0ff4a2f84a0d9945e9b9e4afe32e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/68466811c19b49449285fc2aac90302e, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/670e509773714fb29f4d92cb5dd50abe, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/69f4018db5f3401699474ba8cd85a379, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/12aa7ac787e94e0a9e2fab18d73a91cc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/5080d1eb3d33416eb9705e00808dbec9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/97b0c5532ac14696a4bf4c24330d44ee, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/29d602fd44bf41018f4276c64ccc8caf, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/283f96d53b1e40d3bf62be3261ee7930] to archive 2024-11-15T23:18:08,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:18:08,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/863dcdda3ffe40198d91c3f7a06fa9a0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/863dcdda3ffe40198d91c3f7a06fa9a0 2024-11-15T23:18:08,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/c6d67fbe7a994cc9bf5a3de67482ada0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/c6d67fbe7a994cc9bf5a3de67482ada0 2024-11-15T23:18:08,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/ee090792f9794d97a76ed32fe4ac71e2 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/ee090792f9794d97a76ed32fe4ac71e2 2024-11-15T23:18:08,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/376af1ed0cff4b188f158768566ee302 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/376af1ed0cff4b188f158768566ee302 2024-11-15T23:18:08,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/ab983dae524d400d8afabb9cbc4e4d69 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/ab983dae524d400d8afabb9cbc4e4d69 2024-11-15T23:18:08,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/2ddaed68bc654196a31408ba5f004707 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/2ddaed68bc654196a31408ba5f004707 2024-11-15T23:18:08,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/e101d612c8a243bc9f1025fbd0dc3278 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/e101d612c8a243bc9f1025fbd0dc3278 2024-11-15T23:18:08,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/f32e8e274cde427689d9c5492b0ac52d to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/f32e8e274cde427689d9c5492b0ac52d 2024-11-15T23:18:08,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/f1a5c7dd95fb49d1ac9bcba6f456da34 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/f1a5c7dd95fb49d1ac9bcba6f456da34 2024-11-15T23:18:08,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/9c2d0ff4a2f84a0d9945e9b9e4afe32e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/9c2d0ff4a2f84a0d9945e9b9e4afe32e 2024-11-15T23:18:08,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/68466811c19b49449285fc2aac90302e to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/68466811c19b49449285fc2aac90302e 2024-11-15T23:18:08,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/670e509773714fb29f4d92cb5dd50abe to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/670e509773714fb29f4d92cb5dd50abe 2024-11-15T23:18:08,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/69f4018db5f3401699474ba8cd85a379 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/69f4018db5f3401699474ba8cd85a379 2024-11-15T23:18:08,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/12aa7ac787e94e0a9e2fab18d73a91cc to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/12aa7ac787e94e0a9e2fab18d73a91cc 2024-11-15T23:18:08,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/5080d1eb3d33416eb9705e00808dbec9 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/5080d1eb3d33416eb9705e00808dbec9 2024-11-15T23:18:08,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/97b0c5532ac14696a4bf4c24330d44ee to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/97b0c5532ac14696a4bf4c24330d44ee 2024-11-15T23:18:08,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/29d602fd44bf41018f4276c64ccc8caf to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/29d602fd44bf41018f4276c64ccc8caf 2024-11-15T23:18:08,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/283f96d53b1e40d3bf62be3261ee7930 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/283f96d53b1e40d3bf62be3261ee7930 2024-11-15T23:18:08,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/f134c955845c43f68e65379e668948c7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/f691d2301e4340b690b835eff6fe85c9, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/e7d58a488fdb4e3a9951dea8c2420a62, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/d2f238ffccce4a1eb90ac81ed245ade1, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/004b8216c6504b3e84590888cfc296f5, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/bb8a8e0c5bcc48cabcd03f1f6179b240, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/9c8545fcc3ca4aa49477316821359a1a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/0e717e5e19ce451b87afe7da8fe70ad7, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/b44008aab37f4aceb418a3146b382175, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/7c82e93185774a5fb9bfae8333bb44af, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/02c2c22a928946c6ac5e6a4a2ffa43bc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/cf1e7ae212ea4f459932272d96d3f18f, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/0fe9cb74d9974134b1e2a7654fcb0618, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/6645f844ce4b4f9d9092c02847fbba1b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/1c49fd5a83704af28ab7aae24b29151a, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/a7fc6d0bc618443bb1af9e56ce4b1a2d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/b38fb512553d4f40b3a6686ab867e569, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/974fe27af73b47ef89ca454e7c0754b8] to archive 2024-11-15T23:18:08,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:18:08,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/f134c955845c43f68e65379e668948c7 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/f134c955845c43f68e65379e668948c7 2024-11-15T23:18:08,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/f691d2301e4340b690b835eff6fe85c9 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/f691d2301e4340b690b835eff6fe85c9 2024-11-15T23:18:08,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/e7d58a488fdb4e3a9951dea8c2420a62 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/e7d58a488fdb4e3a9951dea8c2420a62 2024-11-15T23:18:08,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/d2f238ffccce4a1eb90ac81ed245ade1 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/d2f238ffccce4a1eb90ac81ed245ade1 2024-11-15T23:18:08,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/004b8216c6504b3e84590888cfc296f5 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/004b8216c6504b3e84590888cfc296f5 2024-11-15T23:18:08,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/bb8a8e0c5bcc48cabcd03f1f6179b240 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/bb8a8e0c5bcc48cabcd03f1f6179b240 2024-11-15T23:18:08,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/9c8545fcc3ca4aa49477316821359a1a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/9c8545fcc3ca4aa49477316821359a1a 2024-11-15T23:18:08,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/0e717e5e19ce451b87afe7da8fe70ad7 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/0e717e5e19ce451b87afe7da8fe70ad7 2024-11-15T23:18:08,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/b44008aab37f4aceb418a3146b382175 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/b44008aab37f4aceb418a3146b382175 2024-11-15T23:18:08,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/7c82e93185774a5fb9bfae8333bb44af to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/7c82e93185774a5fb9bfae8333bb44af 2024-11-15T23:18:08,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/02c2c22a928946c6ac5e6a4a2ffa43bc to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/02c2c22a928946c6ac5e6a4a2ffa43bc 2024-11-15T23:18:08,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/cf1e7ae212ea4f459932272d96d3f18f to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/cf1e7ae212ea4f459932272d96d3f18f 2024-11-15T23:18:08,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/0fe9cb74d9974134b1e2a7654fcb0618 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/0fe9cb74d9974134b1e2a7654fcb0618 2024-11-15T23:18:08,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/6645f844ce4b4f9d9092c02847fbba1b to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/6645f844ce4b4f9d9092c02847fbba1b 2024-11-15T23:18:08,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/1c49fd5a83704af28ab7aae24b29151a to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/1c49fd5a83704af28ab7aae24b29151a 2024-11-15T23:18:08,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/a7fc6d0bc618443bb1af9e56ce4b1a2d to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/a7fc6d0bc618443bb1af9e56ce4b1a2d 2024-11-15T23:18:08,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/b38fb512553d4f40b3a6686ab867e569 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/b38fb512553d4f40b3a6686ab867e569 2024-11-15T23:18:08,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/974fe27af73b47ef89ca454e7c0754b8 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/974fe27af73b47ef89ca454e7c0754b8 2024-11-15T23:18:08,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/aa8b04c838ae4c93aea5d36d248106a2, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/36bbc13195534ec388ce89e032c19c79, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/f9135a7606ad4cc8affefcbcfab61532, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/bbae5312d3cc4660a926927ced6e4289, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/ef60a25c80294d60961c5637f8405b3b, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/e542ba34d5704e03b6c34aaa204d288d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/ac9caa584ceb4f60b9f39a2904d62ab0, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/bfa00165989b4b9c9eda67a7731768e2, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/3110e18872ce4a2080c46c78844b0a48, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/9241baf91f564beca893158bc4a2b98d, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/d63a3a0a51e3452ebec3e004635f74af, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/ba9ddff0cd6d4a4db68ccd799d16f7a2, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/6914ffa8906a4346ac3545cafdda6202, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/2943638df95841bfb24ca8fc69eced20, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/bea4cd672dc9437fb72a5f3a7c68b706, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/64735b1959f74fc5ab58e1f2338ae8dc, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/e5f0ccb7650e4a22b04eb6e7e3185f77, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/6e6ace6e6bef4761975f5115e4d5bb71] to archive 2024-11-15T23:18:08,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:18:08,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/aa8b04c838ae4c93aea5d36d248106a2 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/aa8b04c838ae4c93aea5d36d248106a2 2024-11-15T23:18:08,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/36bbc13195534ec388ce89e032c19c79 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/36bbc13195534ec388ce89e032c19c79 2024-11-15T23:18:08,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/f9135a7606ad4cc8affefcbcfab61532 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/f9135a7606ad4cc8affefcbcfab61532 2024-11-15T23:18:08,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/bbae5312d3cc4660a926927ced6e4289 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/bbae5312d3cc4660a926927ced6e4289 2024-11-15T23:18:08,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/ef60a25c80294d60961c5637f8405b3b to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/ef60a25c80294d60961c5637f8405b3b 2024-11-15T23:18:08,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/e542ba34d5704e03b6c34aaa204d288d to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/e542ba34d5704e03b6c34aaa204d288d 2024-11-15T23:18:08,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/ac9caa584ceb4f60b9f39a2904d62ab0 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/ac9caa584ceb4f60b9f39a2904d62ab0 2024-11-15T23:18:08,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/bfa00165989b4b9c9eda67a7731768e2 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/bfa00165989b4b9c9eda67a7731768e2 2024-11-15T23:18:08,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/3110e18872ce4a2080c46c78844b0a48 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/3110e18872ce4a2080c46c78844b0a48 2024-11-15T23:18:08,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/9241baf91f564beca893158bc4a2b98d to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/9241baf91f564beca893158bc4a2b98d 2024-11-15T23:18:08,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/d63a3a0a51e3452ebec3e004635f74af to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/d63a3a0a51e3452ebec3e004635f74af 2024-11-15T23:18:08,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/ba9ddff0cd6d4a4db68ccd799d16f7a2 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/ba9ddff0cd6d4a4db68ccd799d16f7a2 2024-11-15T23:18:08,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/6914ffa8906a4346ac3545cafdda6202 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/6914ffa8906a4346ac3545cafdda6202 2024-11-15T23:18:08,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/2943638df95841bfb24ca8fc69eced20 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/2943638df95841bfb24ca8fc69eced20 2024-11-15T23:18:08,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/bea4cd672dc9437fb72a5f3a7c68b706 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/bea4cd672dc9437fb72a5f3a7c68b706 2024-11-15T23:18:08,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/64735b1959f74fc5ab58e1f2338ae8dc to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/64735b1959f74fc5ab58e1f2338ae8dc 2024-11-15T23:18:08,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/e5f0ccb7650e4a22b04eb6e7e3185f77 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/e5f0ccb7650e4a22b04eb6e7e3185f77 2024-11-15T23:18:08,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/6e6ace6e6bef4761975f5115e4d5bb71 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/6e6ace6e6bef4761975f5115e4d5bb71 2024-11-15T23:18:08,335 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/recovered.edits/273.seqid, newMaxSeqId=273, maxSeqId=4 2024-11-15T23:18:08,335 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. 2024-11-15T23:18:08,335 DEBUG [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for 06815b48d79c548c6fc12bf365c2f5fd: Waiting for close lock at 1731712686647Running coprocessor pre-close hooks at 1731712686647Disabling compacts and flushes for region at 1731712686647Disabling writes for close at 1731712686647Obtaining lock to block concurrent updates at 1731712686647Preparing flush snapshotting stores in 06815b48d79c548c6fc12bf365c2f5fd at 1731712686647Finished memstore snapshotting TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731712686648 (+1 ms)Flushing stores of TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd. at 1731712686648Flushing 06815b48d79c548c6fc12bf365c2f5fd/A: creating writer at 1731712686648Flushing mob file 06815b48d79c548c6fc12bf365c2f5fd/A: appending metadata at 1731712686652 (+4 ms)Flushing mob file 06815b48d79c548c6fc12bf365c2f5fd/A: closing flushed file at 1731712686652Flushing 06815b48d79c548c6fc12bf365c2f5fd/A: appending metadata at 1731712687058 (+406 ms)Flushing 06815b48d79c548c6fc12bf365c2f5fd/A: closing flushed file at 1731712687059 (+1 ms)Flushing 06815b48d79c548c6fc12bf365c2f5fd/B: creating writer at 1731712687465 (+406 ms)Flushing 06815b48d79c548c6fc12bf365c2f5fd/B: appending metadata at 1731712687467 (+2 ms)Flushing 06815b48d79c548c6fc12bf365c2f5fd/B: closing flushed file at 1731712687467Flushing 06815b48d79c548c6fc12bf365c2f5fd/C: creating writer at 1731712687874 (+407 ms)Flushing 06815b48d79c548c6fc12bf365c2f5fd/C: appending metadata at 1731712687875 (+1 ms)Flushing 06815b48d79c548c6fc12bf365c2f5fd/C: closing flushed file at 1731712687875Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ad64972: reopening flushed file at 1731712688282 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3644c96a: reopening flushed file at 1731712688285 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@444097f2: reopening flushed file at 1731712688288 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 06815b48d79c548c6fc12bf365c2f5fd in 1645ms, sequenceid=270, compaction requested=true at 1731712688292 (+4 ms)Writing region close event to WAL at 1731712688333 (+41 ms)Running coprocessor post-close hooks at 1731712688335 (+2 ms)Closed at 1731712688335 2024-11-15T23:18:08,336 INFO [RS_CLOSE_REGION-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed 06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:08,337 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=06815b48d79c548c6fc12bf365c2f5fd, regionState=CLOSED 2024-11-15T23:18:08,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 because future has completed 2024-11-15T23:18:08,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-15T23:18:08,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure 06815b48d79c548c6fc12bf365c2f5fd, server=d6ecb9b7dda1,38353,1731712503763 in 1.8430 sec 2024-11-15T23:18:08,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-11-15T23:18:08,341 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=06815b48d79c548c6fc12bf365c2f5fd, UNASSIGN in 1.8460 sec 2024-11-15T23:18:08,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-15T23:18:08,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8500 sec 2024-11-15T23:18:08,343 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731712688343"}]},"ts":"1731712688343"} 2024-11-15T23:18:08,344 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T23:18:08,344 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T23:18:08,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8570 sec 2024-11-15T23:18:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T23:18:08,635 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:18:08,636 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T23:18:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:18:08,638 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:18:08,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-15T23:18:08,639 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:18:08,640 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:08,642 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A, FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B, FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C, FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/recovered.edits] 2024-11-15T23:18:08,644 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/07fc4d5b96fc44c984dd82ac75503124 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/07fc4d5b96fc44c984dd82ac75503124 2024-11-15T23:18:08,645 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/216ffe2f38b34610ad8880f9e186062c to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/216ffe2f38b34610ad8880f9e186062c 2024-11-15T23:18:08,646 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/343494bb8e3a44618ffef0dc41f795c5 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/A/343494bb8e3a44618ffef0dc41f795c5 2024-11-15T23:18:08,648 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/01838482d0214862a037149dfaa11b30 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/01838482d0214862a037149dfaa11b30 2024-11-15T23:18:08,649 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/8a4279e18e6b441e87ac8b102b9bcbd2 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/8a4279e18e6b441e87ac8b102b9bcbd2 2024-11-15T23:18:08,649 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/e27c7d1790f944cfa629a93477c9672b to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/B/e27c7d1790f944cfa629a93477c9672b 2024-11-15T23:18:08,652 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/8753e00c6dde4b608a1e73f768e8b4f8 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/8753e00c6dde4b608a1e73f768e8b4f8 2024-11-15T23:18:08,653 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/9c93c9edf27f4a32b7a0d2a2ade09d09 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/9c93c9edf27f4a32b7a0d2a2ade09d09 2024-11-15T23:18:08,653 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/a0535d8a51974d17ad22a497f7f52262 to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/C/a0535d8a51974d17ad22a497f7f52262 2024-11-15T23:18:08,656 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/recovered.edits/273.seqid to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd/recovered.edits/273.seqid 2024-11-15T23:18:08,657 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/default/TestAcidGuarantees/06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:08,657 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T23:18:08,657 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T23:18:08,657 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T23:18:08,659 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115001adfe8324c42f78b1982c87a0f7cfb_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115001adfe8324c42f78b1982c87a0f7cfb_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:08,660 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115168ed4e55cfe474e8c6882e5629703f5_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115168ed4e55cfe474e8c6882e5629703f5_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:08,660 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115302b974fdf71482690dec2e41c6a65ad_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115302b974fdf71482690dec2e41c6a65ad_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:08,661 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111531aca22e94a74f308f524d80e91240ea_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111531aca22e94a74f308f524d80e91240ea_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:08,662 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115355b63deaf8049b1946427e83a38bec6_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115355b63deaf8049b1946427e83a38bec6_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:08,663 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115395bca1000594cb9b6d465a9b15c5fa2_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115395bca1000594cb9b6d465a9b15c5fa2_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:08,663 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157c5e491880fe4e879d8bea8f82970706_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157c5e491880fe4e879d8bea8f82970706_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:08,664 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158e8a830540424cdabc553a35083a6347_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158e8a830540424cdabc553a35083a6347_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:08,665 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115945b82c3e37d4715851daf75064779b9_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115945b82c3e37d4715851daf75064779b9_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:08,665 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b33e85ac978b409196ae8b048de6e6b7_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b33e85ac978b409196ae8b048de6e6b7_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:08,666 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b91e1f70c9f241a3a5741f69610a6177_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b91e1f70c9f241a3a5741f69610a6177_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:08,667 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115be7730d9b0ce4a5895b7f1ea5ee07759_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115be7730d9b0ce4a5895b7f1ea5ee07759_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:08,667 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c0780f0816c946908e038809df7ddaeb_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c0780f0816c946908e038809df7ddaeb_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:08,668 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c0c46a2d8290401ab9cdab186f40ed9e_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c0c46a2d8290401ab9cdab186f40ed9e_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:08,669 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c194e18cb1d74cd0be086fa8b3cc0d15_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c194e18cb1d74cd0be086fa8b3cc0d15_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:08,670 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115df000f2c71664cba972e6906438c0dcd_06815b48d79c548c6fc12bf365c2f5fd to hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115df000f2c71664cba972e6906438c0dcd_06815b48d79c548c6fc12bf365c2f5fd 2024-11-15T23:18:08,670 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T23:18:08,672 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:18:08,674 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T23:18:08,676 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T23:18:08,677 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:18:08,677 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T23:18:08,677 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731712688677"}]},"ts":"9223372036854775807"} 2024-11-15T23:18:08,678 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T23:18:08,678 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 06815b48d79c548c6fc12bf365c2f5fd, NAME => 'TestAcidGuarantees,,1731712660197.06815b48d79c548c6fc12bf365c2f5fd.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T23:18:08,678 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T23:18:08,679 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731712688678"}]},"ts":"9223372036854775807"} 2024-11-15T23:18:08,680 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T23:18:08,680 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:18:08,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-15T23:18:08,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43421 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-15T23:18:08,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T23:18:08,747 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:18:08,752 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249 (was 248) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=510 (was 387) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5720 (was 6089) 2024-11-15T23:18:08,752 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T23:18:08,752 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T23:18:08,752 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T23:18:08,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:08,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:08,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:18:08,753 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T23:18:08,753 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=350595032, stopped=false 2024-11-15T23:18:08,753 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d6ecb9b7dda1,43421,1731712502855 2024-11-15T23:18:08,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T23:18:08,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T23:18:08,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:18:08,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:18:08,793 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T23:18:08,793 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T23:18:08,793 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T23:18:08,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:08,793 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd6ecb9b7dda1,38353,1731712503763' ***** 2024-11-15T23:18:08,793 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T23:18:08,794 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T23:18:08,794 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T23:18:08,794 INFO [RS:0;d6ecb9b7dda1:38353 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T23:18:08,794 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T23:18:08,794 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T23:18:08,794 INFO [RS:0;d6ecb9b7dda1:38353 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T23:18:08,794 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(959): stopping server d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:08,794 INFO [RS:0;d6ecb9b7dda1:38353 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T23:18:08,794 INFO [RS:0;d6ecb9b7dda1:38353 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d6ecb9b7dda1:38353. 2024-11-15T23:18:08,794 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T23:18:08,794 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:08,795 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T23:18:08,795 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T23:18:08,795 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T23:18:08,795 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T23:18:08,795 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-15T23:18:08,795 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-15T23:18:08,796 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T23:18:08,796 DEBUG [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T23:18:08,796 INFO [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T23:18:08,796 DEBUG [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T23:18:08,796 DEBUG [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T23:18:08,796 DEBUG [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T23:18:08,796 INFO [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-15T23:18:08,815 DEBUG [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/.tmp/info/fdfb9993a1d84840a2914657789d8b5c is 95, key is TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588./info:/1731712537782/DeleteFamily/seqid=0 2024-11-15T23:18:08,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742446_1622 (size=6514) 2024-11-15T23:18:08,850 INFO [regionserver/d6ecb9b7dda1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T23:18:08,996 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T23:18:09,196 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T23:18:09,218 INFO [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/.tmp/info/fdfb9993a1d84840a2914657789d8b5c 2024-11-15T23:18:09,235 DEBUG [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/.tmp/ns/531ae8201c434c51a36481b374677ee0 is 93, key is TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588./ns:/1731712537744/DeleteFamily/seqid=0 2024-11-15T23:18:09,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742447_1623 (size=5906) 2024-11-15T23:18:09,396 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T23:18:09,596 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T23:18:09,639 INFO [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/.tmp/ns/531ae8201c434c51a36481b374677ee0 2024-11-15T23:18:09,654 DEBUG [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/.tmp/rep_barrier/b78691c49a064b67871049f47386b87e is 102, key is TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588./rep_barrier:/1731712537744/DeleteFamily/seqid=0 2024-11-15T23:18:09,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742448_1624 (size=6025) 2024-11-15T23:18:09,796 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T23:18:09,846 INFO [regionserver/d6ecb9b7dda1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-15T23:18:09,846 INFO [regionserver/d6ecb9b7dda1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-15T23:18:09,997 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T23:18:10,057 INFO [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/.tmp/rep_barrier/b78691c49a064b67871049f47386b87e 2024-11-15T23:18:10,073 DEBUG [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/.tmp/table/073931a7f3f74f0d911efefefd20b737 is 96, key is TestAcidGuarantees,,1731712508464.b9ac2b510c393ceae30043c6a9eaa588./table:/1731712537744/DeleteFamily/seqid=0 2024-11-15T23:18:10,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742449_1625 (size=5936) 2024-11-15T23:18:10,197 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T23:18:10,397 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T23:18:10,476 INFO [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/.tmp/table/073931a7f3f74f0d911efefefd20b737 2024-11-15T23:18:10,479 DEBUG [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/.tmp/info/fdfb9993a1d84840a2914657789d8b5c as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/info/fdfb9993a1d84840a2914657789d8b5c 2024-11-15T23:18:10,482 INFO [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/info/fdfb9993a1d84840a2914657789d8b5c, entries=12, sequenceid=90, filesize=6.4 K 2024-11-15T23:18:10,483 DEBUG [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/.tmp/ns/531ae8201c434c51a36481b374677ee0 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/ns/531ae8201c434c51a36481b374677ee0 2024-11-15T23:18:10,485 INFO [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/ns/531ae8201c434c51a36481b374677ee0, entries=8, sequenceid=90, filesize=5.8 K 2024-11-15T23:18:10,486 DEBUG [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/.tmp/rep_barrier/b78691c49a064b67871049f47386b87e as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/rep_barrier/b78691c49a064b67871049f47386b87e 2024-11-15T23:18:10,489 INFO [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/rep_barrier/b78691c49a064b67871049f47386b87e, entries=6, sequenceid=90, filesize=5.9 K 2024-11-15T23:18:10,490 DEBUG [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/.tmp/table/073931a7f3f74f0d911efefefd20b737 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/table/073931a7f3f74f0d911efefefd20b737 2024-11-15T23:18:10,492 INFO [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/table/073931a7f3f74f0d911efefefd20b737, entries=7, sequenceid=90, filesize=5.8 K 2024-11-15T23:18:10,493 INFO [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1697ms, sequenceid=90, compaction requested=false 2024-11-15T23:18:10,498 DEBUG [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-15T23:18:10,498 DEBUG [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T23:18:10,498 INFO [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T23:18:10,498 DEBUG [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731712688795Running coprocessor pre-close hooks at 1731712688796 (+1 ms)Disabling compacts and flushes for region at 1731712688796Disabling writes for close at 1731712688796Obtaining lock to block concurrent updates at 1731712688796Preparing flush snapshotting stores in 1588230740 at 1731712688796Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731712688796Flushing stores of hbase:meta,,1.1588230740 at 1731712688797 (+1 ms)Flushing 1588230740/info: creating writer at 1731712688797Flushing 1588230740/info: appending metadata at 1731712688815 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731712688815Flushing 1588230740/ns: creating writer at 1731712689222 (+407 ms)Flushing 1588230740/ns: appending metadata at 1731712689235 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731712689235Flushing 1588230740/rep_barrier: creating writer at 1731712689641 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731712689654 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731712689654Flushing 1588230740/table: creating writer at 1731712690060 (+406 ms)Flushing 1588230740/table: appending metadata at 1731712690073 (+13 ms)Flushing 1588230740/table: closing flushed file at 1731712690073Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26d6327d: reopening flushed file at 1731712690479 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63d10497: reopening flushed file at 1731712690482 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b297c5f: reopening flushed file at 1731712690485 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@172b3567: reopening flushed file at 1731712690489 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1697ms, sequenceid=90, compaction requested=false at 1731712690493 (+4 ms)Writing region close event to WAL at 1731712690496 (+3 ms)Running coprocessor post-close hooks at 1731712690498 (+2 ms)Closed at 1731712690498 2024-11-15T23:18:10,498 DEBUG [RS_CLOSE_META-regionserver/d6ecb9b7dda1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T23:18:10,597 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(976): stopping server d6ecb9b7dda1,38353,1731712503763; all regions closed. 2024-11-15T23:18:10,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741834_1010 (size=25344) 2024-11-15T23:18:10,603 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/oldWALs 2024-11-15T23:18:10,603 INFO [RS:0;d6ecb9b7dda1:38353 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d6ecb9b7dda1%2C38353%2C1731712503763.meta:.meta(num 1731712508029) 2024-11-15T23:18:10,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741832_1008 (size=15511431) 2024-11-15T23:18:10,606 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/oldWALs 2024-11-15T23:18:10,606 INFO [RS:0;d6ecb9b7dda1:38353 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d6ecb9b7dda1%2C38353%2C1731712503763:(num 1731712506909) 2024-11-15T23:18:10,606 DEBUG [RS:0;d6ecb9b7dda1:38353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:18:10,606 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T23:18:10,606 INFO [RS:0;d6ecb9b7dda1:38353 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T23:18:10,606 INFO [RS:0;d6ecb9b7dda1:38353 {}] hbase.ChoreService(370): Chore service for: regionserver/d6ecb9b7dda1:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-15T23:18:10,606 INFO [RS:0;d6ecb9b7dda1:38353 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T23:18:10,607 INFO [regionserver/d6ecb9b7dda1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T23:18:10,607 INFO [RS:0;d6ecb9b7dda1:38353 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38353 2024-11-15T23:18:10,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T23:18:10,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d6ecb9b7dda1,38353,1731712503763 2024-11-15T23:18:10,671 INFO [RS:0;d6ecb9b7dda1:38353 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T23:18:10,672 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d6ecb9b7dda1,38353,1731712503763] 2024-11-15T23:18:10,687 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d6ecb9b7dda1,38353,1731712503763 already deleted, retry=false 2024-11-15T23:18:10,688 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d6ecb9b7dda1,38353,1731712503763 expired; onlineServers=0 2024-11-15T23:18:10,688 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd6ecb9b7dda1,43421,1731712502855' ***** 2024-11-15T23:18:10,688 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T23:18:10,688 INFO [M:0;d6ecb9b7dda1:43421 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T23:18:10,688 INFO [M:0;d6ecb9b7dda1:43421 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T23:18:10,688 DEBUG [M:0;d6ecb9b7dda1:43421 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T23:18:10,688 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T23:18:10,688 DEBUG [M:0;d6ecb9b7dda1:43421 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T23:18:10,688 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster-HFileCleaner.small.0-1731712506645 {}] cleaner.HFileCleaner(306): Exit Thread[master/d6ecb9b7dda1:0:becomeActiveMaster-HFileCleaner.small.0-1731712506645,5,FailOnTimeoutGroup] 2024-11-15T23:18:10,688 DEBUG [master/d6ecb9b7dda1:0:becomeActiveMaster-HFileCleaner.large.0-1731712506643 {}] cleaner.HFileCleaner(306): Exit Thread[master/d6ecb9b7dda1:0:becomeActiveMaster-HFileCleaner.large.0-1731712506643,5,FailOnTimeoutGroup] 2024-11-15T23:18:10,689 INFO [M:0;d6ecb9b7dda1:43421 {}] hbase.ChoreService(370): Chore service for: master/d6ecb9b7dda1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T23:18:10,689 INFO [M:0;d6ecb9b7dda1:43421 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T23:18:10,689 DEBUG [M:0;d6ecb9b7dda1:43421 {}] master.HMaster(1795): Stopping service threads 2024-11-15T23:18:10,689 INFO [M:0;d6ecb9b7dda1:43421 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T23:18:10,689 INFO [M:0;d6ecb9b7dda1:43421 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T23:18:10,689 ERROR [M:0;d6ecb9b7dda1:43421 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-15T23:18:10,690 INFO [M:0;d6ecb9b7dda1:43421 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T23:18:10,690 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T23:18:10,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T23:18:10,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:18:10,698 DEBUG [M:0;d6ecb9b7dda1:43421 {}] zookeeper.ZKUtil(347): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-15T23:18:10,698 WARN [M:0;d6ecb9b7dda1:43421 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-15T23:18:10,699 INFO [M:0;d6ecb9b7dda1:43421 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/.lastflushedseqids 2024-11-15T23:18:10,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742450_1626 (size=72) 2024-11-15T23:18:10,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T23:18:10,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x10140c7c4b40001, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T23:18:10,788 INFO [RS:0;d6ecb9b7dda1:38353 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T23:18:10,789 INFO [RS:0;d6ecb9b7dda1:38353 {}] regionserver.HRegionServer(1031): Exiting; stopping=d6ecb9b7dda1,38353,1731712503763; zookeeper connection closed. 2024-11-15T23:18:10,789 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3a9c13ea {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3a9c13ea 2024-11-15T23:18:10,789 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-15T23:18:11,110 INFO [M:0;d6ecb9b7dda1:43421 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T23:18:11,110 INFO [M:0;d6ecb9b7dda1:43421 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T23:18:11,110 DEBUG [M:0;d6ecb9b7dda1:43421 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T23:18:11,110 INFO [M:0;d6ecb9b7dda1:43421 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T23:18:11,110 DEBUG [M:0;d6ecb9b7dda1:43421 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T23:18:11,110 DEBUG [M:0;d6ecb9b7dda1:43421 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T23:18:11,110 DEBUG [M:0;d6ecb9b7dda1:43421 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T23:18:11,110 INFO [M:0;d6ecb9b7dda1:43421 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=774.53 KB heapSize=949.02 KB 2024-11-15T23:18:11,124 DEBUG [M:0;d6ecb9b7dda1:43421 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/55b449768f4d4a7d90b17ac6b749230b is 82, key is hbase:meta,,1/info:regioninfo/1731712508115/Put/seqid=0 2024-11-15T23:18:11,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742451_1627 (size=5672) 2024-11-15T23:18:11,527 INFO [M:0;d6ecb9b7dda1:43421 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2155 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/55b449768f4d4a7d90b17ac6b749230b 2024-11-15T23:18:11,546 DEBUG [M:0;d6ecb9b7dda1:43421 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/356aa26fd74e479690afb36217968874 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x93/proc:d/1731712663299/Put/seqid=0 2024-11-15T23:18:11,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742452_1628 (size=45348) 2024-11-15T23:18:11,949 INFO [M:0;d6ecb9b7dda1:43421 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=773.97 KB at sequenceid=2155 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/356aa26fd74e479690afb36217968874 2024-11-15T23:18:11,952 INFO [M:0;d6ecb9b7dda1:43421 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 356aa26fd74e479690afb36217968874 2024-11-15T23:18:11,971 DEBUG [M:0;d6ecb9b7dda1:43421 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c5c153657743496993dab10fb03a20f9 is 69, key is d6ecb9b7dda1,38353,1731712503763/rs:state/1731712506672/Put/seqid=0 2024-11-15T23:18:11,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073742453_1629 (size=5156) 2024-11-15T23:18:12,374 INFO [M:0;d6ecb9b7dda1:43421 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2155 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c5c153657743496993dab10fb03a20f9 2024-11-15T23:18:12,377 DEBUG [M:0;d6ecb9b7dda1:43421 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/55b449768f4d4a7d90b17ac6b749230b as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/55b449768f4d4a7d90b17ac6b749230b 2024-11-15T23:18:12,380 INFO [M:0;d6ecb9b7dda1:43421 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/55b449768f4d4a7d90b17ac6b749230b, entries=8, sequenceid=2155, filesize=5.5 K 2024-11-15T23:18:12,381 DEBUG [M:0;d6ecb9b7dda1:43421 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/356aa26fd74e479690afb36217968874 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/356aa26fd74e479690afb36217968874 2024-11-15T23:18:12,383 INFO [M:0;d6ecb9b7dda1:43421 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 356aa26fd74e479690afb36217968874 2024-11-15T23:18:12,383 INFO [M:0;d6ecb9b7dda1:43421 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/356aa26fd74e479690afb36217968874, entries=170, sequenceid=2155, filesize=44.3 K 2024-11-15T23:18:12,384 DEBUG [M:0;d6ecb9b7dda1:43421 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c5c153657743496993dab10fb03a20f9 as hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c5c153657743496993dab10fb03a20f9 2024-11-15T23:18:12,387 INFO [M:0;d6ecb9b7dda1:43421 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34017/user/jenkins/test-data/41718e5e-7575-5f58-863a-e8e93f8de444/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c5c153657743496993dab10fb03a20f9, entries=1, sequenceid=2155, filesize=5.0 K 2024-11-15T23:18:12,387 INFO [M:0;d6ecb9b7dda1:43421 {}] regionserver.HRegion(3140): Finished flush of dataSize ~774.53 KB/793114, heapSize ~948.73 KB/971496, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1277ms, sequenceid=2155, compaction requested=false 2024-11-15T23:18:12,388 INFO [M:0;d6ecb9b7dda1:43421 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T23:18:12,388 DEBUG [M:0;d6ecb9b7dda1:43421 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731712691110Disabling compacts and flushes for region at 1731712691110Disabling writes for close at 1731712691110Obtaining lock to block concurrent updates at 1731712691110Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731712691110Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=793114, getHeapSize=971736, getOffHeapSize=0, getCellsCount=2350 at 1731712691110Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731712691111 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731712691111Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731712691124 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731712691124Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731712691530 (+406 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731712691545 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731712691545Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731712691952 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731712691970 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731712691970Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a37227c: reopening flushed file at 1731712692377 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@111b5c6a: reopening flushed file at 1731712692380 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@694a05c4: reopening flushed file at 1731712692383 (+3 ms)Finished flush of dataSize ~774.53 KB/793114, heapSize ~948.73 KB/971496, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1277ms, sequenceid=2155, compaction requested=false at 1731712692387 (+4 ms)Writing region close event to WAL at 1731712692388 (+1 ms)Closed at 1731712692388 2024-11-15T23:18:12,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37491 is added to blk_1073741830_1006 (size=934388) 2024-11-15T23:18:12,391 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T23:18:12,391 INFO [M:0;d6ecb9b7dda1:43421 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-15T23:18:12,391 INFO [M:0;d6ecb9b7dda1:43421 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43421 2024-11-15T23:18:12,391 INFO [M:0;d6ecb9b7dda1:43421 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T23:18:12,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T23:18:12,545 INFO [M:0;d6ecb9b7dda1:43421 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T23:18:12,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43421-0x10140c7c4b40000, quorum=127.0.0.1:65035, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T23:18:12,550 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T23:18:12,551 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T23:18:12,551 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T23:18:12,552 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T23:18:12,552 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/hadoop.log.dir/,STOPPED} 2024-11-15T23:18:12,554 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-15T23:18:12,554 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-15T23:18:12,554 WARN [BP-674942766-172.17.0.2-1731712499119 heartbeating to localhost/127.0.0.1:34017 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-15T23:18:12,554 WARN [BP-674942766-172.17.0.2-1731712499119 heartbeating to localhost/127.0.0.1:34017 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-674942766-172.17.0.2-1731712499119 (Datanode Uuid a6f950b1-7c3b-49b3-8987-cb2fd4e6642f) service to localhost/127.0.0.1:34017 2024-11-15T23:18:12,556 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/cluster_535d8ff1-f3e7-a1c2-84cc-d10f4f48cd95/data/data1/current/BP-674942766-172.17.0.2-1731712499119 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T23:18:12,556 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/cluster_535d8ff1-f3e7-a1c2-84cc-d10f4f48cd95/data/data2/current/BP-674942766-172.17.0.2-1731712499119 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T23:18:12,557 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-15T23:18:12,567 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T23:18:12,568 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T23:18:12,568 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T23:18:12,568 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T23:18:12,568 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/71ad3f98-152f-26a2-9a0c-799af6a1b41f/hadoop.log.dir/,STOPPED} 2024-11-15T23:18:12,581 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-15T23:18:12,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down